def __init__(self, key='', secret=''): self.url = 'https://api.cryptsy.com/api' self.key = key self.secret = secret self.markets = None self.rate_limiter = TokenBucket(RATE) random.seed(time.time())
def __init__(self, *, torrent: state.Torrent, complete_pieces_to_write: trio.MemorySendChannel, write_confirmations: trio.MemoryReceiveChannel, blocks_to_read: trio.MemorySendChannel, blocks_for_peers: trio.MemoryReceiveChannel, auto_shutdown=False) -> None: self._auto_shutdown = auto_shutdown self._state = torrent # interact with self self._peers_without_connection = trio.open_memory_channel( config.INTERNAL_QUEUE_SIZE) # interact with FileManager self._complete_pieces_to_write = complete_pieces_to_write self._write_confirmations = write_confirmations self._blocks_to_read = blocks_to_read self._blocks_for_peers = blocks_for_peers # interact with peer connections self._msg_from_peer = trio.open_memory_channel( config.INTERNAL_QUEUE_SIZE) # queues for sending TO peers are initialized on a per-peer basis self._peers: Dict[bytes, peer_state.PeerState] = dict() # data received but not written to disk self._received_blocks: Dict[int, Tuple[bitarray, bytearray]] = dict() self.requests = requests.RequestManager() if config.MAX_OUTGOING_BYTES_PER_SECOND is None: self.token_bucket: Union[NullBucket, TokenBucket] = NullBucket() else: self.token_bucket = TokenBucket( config.MAX_OUTGOING_BYTES_PER_SECOND)
def generateRouters(graphSizeX, graphSizeY, areaSize, routerChancePerArea, rate, capacity, calendarQueue): routerList = {} routerId = 0 # generates routers homogeneously in a grid while avoiding generating routers to close from each other for i in range(graphSizeX//areaSize): for j in range(graphSizeY//areaSize): if (random.random() < routerChancePerArea): storage = MemoryStorage() bucket = TokenBucket(rate, capacity, storage) x = int((random.random()*0.5 + i + 0.25) * areaSize) y = int((random.random()*0.5 + j + 0.25) * areaSize) router = Router(routerId, x, y, state=True, tokenBucket=bucket, neighbours={}, LSDB={}, bufferSize=10, calendar=calendarQueue, linkStates=None) routerList[routerId] = router routerId+=1 return routerList
def worker(rank, size, input_file_specs, batch_size=256, warmup_sec=10.0, run_sec=60 * 60 * 4, num_threads=0, sync=False, warn_latency_sec=4.0, report_period_sec=2.0, round_robin_files=True, throttle_sleep_sec=0.01, throttle_total_rate_bytes_per_sec=0): if rank == 0: print('storage_benchmark_tensorflow: BEGIN') print(datetime.datetime.utcnow()) metrics_file_name = '/imagenet-scratch/logs/storage_benchmark_tensorflow_metrics-%d.log' % rank with open(metrics_file_name, 'a') as metrics_file: hostname = socket.gethostname() # Set random seed to have deterministic behavior. tf.set_random_seed(rank + 1) # Round robin the input file spec. This allows multiple mount points to be used. input_file_spec = input_file_specs[hvd.local_rank() % len(input_file_specs)] print('rank=%3d: %s: input_file_spec=%s' % (rank, hostname, input_file_spec)) if round_robin_files: # Distribute sets of file names evenly over all processes and without overlap. all_input_filenames = sorted(glob.glob(input_file_spec)) num_files = len(all_input_filenames) i = rank input_filenames = [] while i < num_files: input_filenames.append(all_input_filenames[i]) i += size print( 'rank=%3d: Found %d total files. %d files assigned to this process.' % (rank, len(all_input_filenames), len(input_filenames))) if len(input_filenames) == 0: raise ValueError('Not enough matching files.') input_file_spec = None else: # This will use tf.data.TFRecordDataset.list_files to randomly distribute files. input_filenames = None # # Build execution graph. # ds_iterator = create_iterator(batch_size, num_threads, input_file_spec=input_file_spec, input_filenames=input_filenames) # num_bytes_tensor is an int64 tensor of shape (batch_size). num_bytes_tensor = ds_iterator.get_next() # When num_bytes_for_step_tensor is evaluated, it reads the TFRecord files. num_bytes_for_step_tensor = tf.reduce_sum(num_bytes_tensor) # The following operations are used to synchronize the processes when running in sync mode. if sync: stop_flag_placeholder = tf.placeholder(tf.bool, shape=()) stop_flag_broadcast_tensor = hvd.broadcast(stop_flag_placeholder, 0, 'stop_flag_broadcast') num_bytes_for_step_placeholder = tf.placeholder(tf.int64, shape=()) total_bytes_for_step_tensor = hvd.allreduce( num_bytes_for_step_placeholder, average=False) # # Start the TensorFlow session and execute the graph. # config = tf.ConfigProto() config.device_count['GPU'] = 0 config.intra_op_parallelism_threads = 1 config.inter_op_parallelism_threads = 1 print('rank=%3d: Creating session' % rank) with tf.Session(config=config) as session: print('rank=%3d: Session created' % rank) session.run( [tf.initializers.global_variables(), tf.tables_initializer()]) print('rank=%3d: Initialized variables' % rank) # Run first step. This can take 30 seconds for 100,000 files. print('rank=%3d: Running first step' % rank) _ = session.run(num_bytes_for_step_tensor) print('rank=%3d: First step complete' % rank) # Wait for barrier so we know when all processes have finished the first step. print('rank=%3d: Waiting for barrier' % rank) session.run(hvd.allreduce(tf.constant(0))) if rank == 0: print('rank=%3d: Completed waiting for barrier' % rank) # To ensure that all processes finish warmup and stop at exactly the same time, # the rank 0 node broadcasts its time to all other ranks. # This also serves as a synchronization barrier. local_t0 = time.time() t0_tensor = tf.constant(local_t0, tf.float64) t0_tensor = hvd.broadcast(t0_tensor, 0, 't0') t0 = session.run(t0_tensor) start_time = t0 + warmup_sec stop_time = start_time + run_sec step = 0 warmed_up = False num_records = 0 num_bytes = 0 total_bytes = 0 next_report_time = time.time() + report_period_sec if throttle_total_rate_bytes_per_sec: throttle_rate_bytes_per_sec = throttle_total_rate_bytes_per_sec / size burst_sec = 1.0 throttle = TokenBucket(tokens=throttle_rate_bytes_per_sec * burst_sec, fill_rate=throttle_rate_bytes_per_sec) else: throttle = None while True: # Reset all counters when warmup completes. t = time.time() if not warmed_up and t >= start_time: print('rank=%3d: warmup complete at step %d' % (rank, step)) warmed_up = True t0 = start_time step = 0 num_records = 0 num_bytes = 0 total_bytes = 0 # Run a single step of batch_size records per process. run_options = tf.RunOptions() # run_options.timeout_in_ms = 10000 num_bytes_for_step = np.int64(0) try: num_bytes_for_step = session.run(num_bytes_for_step_tensor, options=run_options) except Exception as e: print('rank=%3d: %s: ERROR: %s' % (rank, hostname, e)) step_dt = time.time() - t if (warmed_up or step >= 1) and step_dt > warn_latency_sec: print('rank=%3d: %s: WARNING: step %d took %0.3f seconds' % (rank, hostname, step, step_dt)) next_report_time = 0.0 # Calculate local stop flag. In sync mode, this is broadcast from rank 0. stop_flag = time.time() >= stop_time # Use Horovod to aggregate the byte counter across all processes. # This also acts as a synchronization barrier, much like gradient descent when # it shares gradients. # Also coordinate the stop flag so all processes stop at the same step. sync_dt = 0.0 if sync: t = time.time() total_bytes_for_step, stop_flag = session.run( [ total_bytes_for_step_tensor, stop_flag_broadcast_tensor ], feed_dict={ num_bytes_for_step_placeholder: num_bytes_for_step, stop_flag_placeholder: stop_flag, }, ) total_bytes += total_bytes_for_step sync_dt = time.time() - t if warmed_up and sync_dt > 30.0: print( 'rank=%3d: %s: WARNING: sync after step %d took %0.3f seconds' % (rank, hostname, step, sync_dt)) next_report_time = 0.0 num_records += batch_size num_bytes += num_bytes_for_step t = time.time() metrics = { '@timestamp': datetime.datetime.utcnow().isoformat() + 'Z', 'batch_size': batch_size, 'rank': rank, 'hostname': hostname, 'step': step, 'num_bytes': int(num_bytes_for_step), 'latency_sec': step_dt, 'sync_latency_sec': sync_dt, } json.dump(metrics, metrics_file) metrics_file.write("\n") metrics_file.flush() if t >= next_report_time: dt = t - t0 if not sync: records_per_sec = num_records / dt bytes_per_sec = num_bytes / dt MB_per_sec = bytes_per_sec / 1e6 print( 'rank=%3d: warmed_up=%d, step=%6d, records/sec=%8.0f, MB/sec=%11.3f, records=%10d, bytes=%15d, dt=%9.3f' % (rank, warmed_up, step, records_per_sec, MB_per_sec, num_records, num_bytes, dt)) if sync: if rank == 0: total_records = num_records * size records_per_sec = total_records / dt bytes_per_sec = total_bytes / dt MB_per_sec = bytes_per_sec / 1e6 print( 'TOTAL: warmed up=%d, step=%6d, records/sec=%8.0f, MB/sec=%11.3f, records=%10d, bytes=%15d, dt=%9.3f' % (warmed_up, step, records_per_sec, MB_per_sec, total_records, total_bytes, dt)) next_report_time = t + report_period_sec # Throttle byte rate. if throttle: while not throttle.consume(num_bytes_for_step): # print('sleeping') time.sleep(throttle_sleep_sec) if stop_flag: print('rank=%3d: %s: complete at step %d' % (rank, hostname, step)) break step += 1 # Use Horovod to aggregate the final counters across all processes. num_steps_tensor = tf.constant(step) num_bytes_tensor = tf.constant(num_bytes) total_steps_tensor = hvd.allreduce(num_steps_tensor, average=False) total_bytes_tensor = hvd.allreduce(num_bytes_tensor, average=False) total_steps, total_bytes = session.run( [total_steps_tensor, total_bytes_tensor]) if rank == 0: dt = stop_time - start_time num_records = total_steps * batch_size records_per_sec = num_records / dt total_GB = total_bytes / 1e9 bytes_per_sec = total_bytes / dt MB_per_sec = bytes_per_sec / 1e6 print('FINAL: number of processes: %12d' % size) print('FINAL: batch size: %12d' % batch_size) print('FINAL: sync: %12s' % sync) print('FINAL: round robin files: %12s' % round_robin_files) print('FINAL: number of records: %12d' % num_records) print('FINAL: GB: %12.3f' % total_GB) print('FINAL: elapsed sec: %12.3f' % dt) print('FINAL: records/sec: %12.0f' % records_per_sec) print('FINAL: MB/sec: %12.3f' % MB_per_sec) if rank == 0: print('storage_benchmark_tensorflow: END')
class api(object): def __init__(self, key='', secret=''): self.url = 'https://api.cryptsy.com/api' self.key = key self.secret = secret self.markets = None self.rate_limiter = TokenBucket(RATE) random.seed(time.time()) def __query(self, method, params={}): req = params req['method'] = method req['nonce'] = int(time.time()) data = urllib.urlencode(req) sign = hmac.new(self.secret, data, hashlib.sha512).hexdigest() headers = { 'Sign': sign, 'Key': self.key } while not self.rate_limiter.may_i(): time.sleep(random.uniform(0.5, 3.0)) r = requests.post(self.url, data=data, headers=headers) if r.status_code == 200: c = json.loads(r.content) if c['success'] == u'1': if method == 'createorder': return c['orderid'] return c['return'] else: log.error('Error calling %s(%s) on %s', method, params, self.url) log.error(c) log.error(c['error']) else: log.error('Error calling %s(%s) on %s', method, params, self.url) return None def getinfo(self): """ Outputs: balances_available Array of currencies and the balances availalbe for each balances_hold Array of currencies and the amounts currently on hold for open orders servertimestamp Current server timestamp servertimezone Current timezone for the server serverdatetime Current date/time on the server openordercount Count of open orders on your account """ return self.__query('getinfo') def getmarkets(self): """ Outputs: Array of Active Markets marketid Integer value representing a market label Name for this market, for example: AMC/BTC primary_currency_code Primary currency code, for example: AMC primary_currency_name Primary currency name, for example: AmericanCoin secondary_currency_code Secondary currency code, for example: BTC secondary_currency_name Secondary currency name, for example: BitCoin current_volume 24 hour trading volume in this market last_trade Last trade price for this market high_trade 24 hour highest trade price in this market low_trade 24 hour lowest trade price in this market created Datetime (EST) the market was created """ return self.__query('getmarkets') def getwalletstatus(self): """ Outputs: Array of Wallet Statuses currencyid Integer value representing a currency name Name for this currency, for example: Bitcoin code Currency code, for example: BTC blockcount Blockcount of currency hot wallet as of lastupdate time difficulty Difficulty of currency hot wallet as of lastupdate time version Version of currency hotwallet as of lastupdate time peercount Connected peers of currency hot wallet as of lastupdate time hashrate Network hashrate of currency hot wallet as of lastupdate time gitrepo Git Repo URL for this currency withdrawalfee Fee charged for withdrawals of this currency lastupdate Datetime (EST) the hot wallet information was last updated """ return self.__query('getwalletstatus') def mytransactions(self): """ Outputs: Array of Deposits and Withdrawals on your account currency Name of currency account timestamp The timestamp the activity posted datetime The datetime the activity posted timezone Server timezone type Type of activity. (Deposit / Withdrawal) address Address to which the deposit posted or Withdrawal was sent amount Amount of transaction (Not including any fees) fee Fee (If any) Charged for this Transaction (Generally only on Withdrawals) trxid Network Transaction ID (If available) """ return self.__query('mytransactions') def markettrades(self, marketid): """ Inputs: marketid Market ID for which you are querying Outputs: Array of last 1000 Trades for this Market, in Date Decending Order tradeid A unique ID for the trade datetime Server datetime trade occurred tradeprice The price the trade occurred at quantity Quantity traded total Total value of trade (tradeprice * quantity) initiate_ordertype The type of order which initiated this trade """ return self.__query('markettrades', params={'marketid': marketid}) def marketorders(self, marketid): """ Inputs: marketid Market ID for which you are querying Outputs: 2 Arrays. First array is sellorders listing current open sell orders ordered price ascending. Second array is buyorders listing current open buy orders ordered price descending. sellprice If a sell order, price which order is selling at buyprice If a buy order, price the order is buying at quantity Quantity on order total Total value of order (price * quantity) """ return self.__query('marketorders', params={'marketid': marketid}) def mytrades(self, marketid, limit=200): """ Inputs: marketid Market ID for which you are querying limit (optional) Limit the number of results. Default: 200 Outputs: Array your Trades for this Market, in Date Decending Order tradeid An integer identifier for this trade tradetype Type of trade (Buy/Sell) datetime Server datetime trade occurred tradeprice The price the trade occurred at quantity Quantity traded total Total value of trade (tradeprice * quantity) - Does not include fees fee Fee Charged for this Trade initiate_ordertype The type of order which initiated this trade order_id Original order id this trade was executed against """ return self.__query('marketorders', params={'marketid': marketid, 'limit': limit}) def allmytrades(self, startdate, enddate): """ Inputs: startdate (optional) Starting date for query (format: yyyy-mm-dd) enddate (optional) Ending date for query (format: yyyy-mm-dd) Outputs: Array your Trades for all Markets, in Date Decending Order tradeid An integer identifier for this trade tradetype Type of trade (Buy/Sell) datetime Server datetime trade occurred marketid The market in which the trade occurred tradeprice The price the trade occurred at quantity Quantity traded total Total value of trade (tradeprice * quantity) - Does not include fees fee Fee Charged for this Trade initiate_ordertype The type of order which initiated this trade order_id Original order id this trade was executed against """ return self.__query('allmytrades', params={'startdate': startdate, 'enddate': enddate}) def myorders(self, marketid): """ Inputs: marketid Market ID for which you are querying Outputs: Array of your orders for this market listing your current open sell and buy orders. orderid Order ID for this order created Datetime the order was created ordertype Type of order (Buy/Sell) price The price per unit for this order quantity Quantity remaining for this order total Total value of order (price * quantity) orig_quantity Original Total Order Quantity """ return self.__query('myorders', params={'marketid': marketid}) def depth(self, marketid): """ Inputs: marketid Market ID for which you are querying Outputs: Array of buy and sell orders on the market representing market depth. Output Format is: array( 'sell'=>array( array(price,quantity), array(price,quantity), .... ), 'buy'=>array( array(price,quantity), array(price,quantity), .... ) ) """ return self.__query('depth', params={'marketid': marketid}) def allmyorders(self): """ Outputs: Array of all open orders for your account. orderid Order ID for this order marketid The Market ID this order was created for created Datetime the order was created ordertype Type of order (Buy/Sell) price The price per unit for this order quantity Quantity remaining for this order total Total value of order (price * quantity) orig_quantity Original Total Order Quantity """ return self.__query('allmyorders') def createorder(self, marketid, ordertype, quantity, price): """ Inputs: marketid Market ID for which you are creating an order for ordertype Order type you are creating (Buy/Sell) quantity Amount of units you are buying/selling in this order price Price per unit you are buying/selling at Outputs: orderid If successful, the Order ID for the order which was created """ return self.__query('createorder', params={ 'marketid': marketid, 'ordertype': ordertype, 'quantity': quantity, 'price': price } ) def cancelorder(self, orderid): """ Inputs: orderid Order ID for which you would like to cancel """ return self.__query('cancelorder', params={'orderid': orderid}) def cancelmarketorders(self, marketid): """ Inputs: marketid Market ID for which you would like to cancel all open orders Outputs: return Array for return information on each order cancelled """ return self.__query('cancelmarketorders', params={'marketid': marketid}) def cancelallorders(self): """ Inputs: N/A Outputs: return Array for return information on each order cancelled """ return self.__query('cancelallorders') def calculatefees(self, ordertype, quantity, price): """ Inputs: ordertype Order type you are calculating for (Buy/Sell) quantity Amount of units you are buying/selling price Price per unit you are buying/selling at Outputs: fee The that would be charged for provided inputs net The net total with fees """ return self.__query('calculatefees', params={'ordertype': ordertype, 'quantity': quantity, 'price': price }) def generatenewaddress(self, currencyid, currencycode): """ Inputs: (either currencyid OR currencycode required - you do not have to supply both) currencyid Currency ID for the coin you want to generate a new address for (ie. 3 = BitCoin) currencycode Currency Code for the coin you want to generate a new address for (ie. BTC = BitCoin) Outputs: address The new generated address """ return self.__query('generatewalletaddress', params={'currencyid': currencyid, 'currencycode': currencycode, }) def mytransfers(self): """ Inputs: n/a Outputs: Array of all transfers into/out of your account sorted by requested datetime descending. currency Currency being transfered request_timestamp Datetime the transfer was requested/initiated processed Indicator if transfer has been processed (1) or not (0) processed_timestamp Datetime of processed transfer from Username sending transfer to Username receiving transfer quantity Quantity being transfered direction Indicates if transfer is incoming or outgoing (in/out) """ return self.__query('mytransfers') def makewithdrawal(self, address, amount): """ Inputs: address Pre-approved Address for which you are withdrawing to (Set up these addresses on Settings page) amount Amount you are withdrawing. Supports up to 8 decimal places. Outputs: Either successful or error. If error, gives reason for error. """ return self.__query('makewithdrawal', params={'address': address, 'amount': amount}) def getmydepositaddresses(self): """ Inputs: n/a Outputs: return Array for return information on each order cancelled ("coincode" => "despositaddress") """ return self.__query('getmydepositaddresses') def getorderstatus(self, orderid): """ Inputs: orderid Order ID for which you are querying Outputs: tradeinfo is a list of all the trades that have occured in your order. Where orderinfo shows realtime status of the order. Orderinfo contains the 'active'; a boolean object showing if the order is still open. Orderinfo also contains 'remainqty' which shows the quantity left in your order. tradeinfo A list of all trades that have occuried in your order. orderinfo Information regarding status of the order. Contains 'active' and 'remainqty' keys. """ return self.__query('getorderstatus', params={'orderid': orderid}) def ex_coin_rev(self, buy_symbol, sell_symbol, sell_volume): orderids = [] log.info('selling %.8f %s for %s', sell_volume, sell_symbol, buy_symbol) if not self.markets: self.markets = self.getmarkets() market_id = '' for market in self.markets: if market['primary_currency_code'] == buy_symbol and \ market['secondary_currency_code'] == sell_symbol: log.info(market) market_id = market['marketid'] break assert(market_id) # get buy orders for ltc depth = self.depth(market_id) bought = 0.0 remaining_sell_volume = sell_volume for order in depth['sell']: log.debug(order) order_volume_to_sell = float(order[1]) order_price_per_coin = float(order[0]) total_price = order_volume_to_sell * order_price_per_coin log.debug('Order %.8f %s for %.8f per coin', order_volume_to_sell, sell_symbol, order_price_per_coin) if remaining_sell_volume < total_price: order_volume_to_sell = remaining_sell_volume / order_price_per_coin #fees = float(self.calculatefees('Sell', order_volume_to_sell, order_price_per_coin)['fee']) # TODO fees = order_volume_to_sell * order_price_per_coin * 0.002 log.debug('We buy %.8f %s for %.8f per coin (total btc %0.8f)', order_volume_to_sell, buy_symbol, order_price_per_coin, order_volume_to_sell * order_price_per_coin) orderid = self.createorder(market_id, 'Buy', order_volume_to_sell, order_price_per_coin) if not orderid: log.error('could not set order') else: orderids.append(orderid) bought += order_volume_to_sell - fees remaining_sell_volume -= total_price if remaining_sell_volume <= 0: break #return (bought, remaining_sell_volume) return orderids def ex_coin(self, buy_symbol, sell_symbol, sell_volume): orderids = [] log.info('selling %.8f %s for %s', sell_volume, sell_symbol, buy_symbol) if not self.markets: self.markets = self.getmarkets() market_id = '' for market in self.markets: if market['primary_currency_code'] == sell_symbol and \ market['secondary_currency_code'] == buy_symbol: log.info(market) market_id = market['marketid'] break assert(market_id) # get buy orders for ltc depth = self.depth(market_id) bought = 0.0 remaining_sell_volume = sell_volume for order in depth['buy']: log.debug(order) order_volume_to_sell = float(order[1]) order_price_per_coin = float(order[0]) log.debug('Order %f %s for %f per coin', order_volume_to_sell, sell_symbol, order_price_per_coin) if remaining_sell_volume < order_volume_to_sell: order_volume_to_sell = remaining_sell_volume #fees = float(self.calculatefees('Sell', order_volume_to_sell, order_price_per_coin)['fee']) # TODO fees = order_volume_to_sell * order_price_per_coin * 0.002 orderid = self.createorder(market_id, 'Sell', order_volume_to_sell, order_price_per_coin) if not orderid: log.error('could not set order') else: orderids.append(orderid) bought += order_volume_to_sell * order_price_per_coin - fees remaining_sell_volume -= order_volume_to_sell if remaining_sell_volume <= 0: break #return (bought, remaining_sell_volume) return orderids
def btn_aniadir(): n = int(en_n.get()) t = int(en_t.get()) my_pattern.append((n, t)) scrolled_text.configure(state='normal') scrolled_text.insert( 'insert', '[*] ' + str(n) + ' Paquetes de ' + str(t) + ' bytes' + '\n') scrolled_text.configure(state='disabled') if __name__ == '__main__': tb = TokenBucket() #tb.set_pattern([(800,8),(600,28),(500,6),(500,25),(2000,8),(400,25)]) my_pattern = [] window = Tk() window.title("Token Bucket") window.geometry('1000x400') numero_de_paquetes = 0 radio_var = IntVar() btn = Button(window, text="Empezar", command=btn_empezar) btn.grid(column=90, row=90) lbl_aue = Label(window, text="Añade un elemento:",
self.scheduler.enter(process_time, 1, router.process_packet) if __name__ == "__main__": listStorage = [] listTocken = [] listRouter = [] rate = 10 capacity = 100 simu = SimulatedTime(0) calendarQueue = CalendarQueue(simu) for i in range(10): listStorage.append(MemoryStorage()) listTocken.append(TokenBucket(rate, capacity, listStorage[i])) listRouter.append( Router(id=i, x=0, y=0, state=True, tokenBucket=listTocken[i], neighbours={}, LSDB={}, bufferSize=10, calendar=calendarQueue, linkStates=None)) if (i > 2): listRouter[i].add_neighbour(2, i) listRouter[2].add_neighbour(i, i)
def __init__(self, *args, **kwargs): super(Detection, self).__init__(*args, **kwargs) random.seed() sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) sys.stdin = os.fdopen(sys.stdin.fileno(), 'r', 0) print >> sys.stderr, 'Loading topology' self.read_config() self.rules = load_rules(Detection.FILENAME + '.rules.pkl') self.topology_graph = load_topology(Detection.FILENAME + '.graphml') self.testpackets = load_testpackets(Detection.FILENAME + '.testpackets.pkl') self.datapaths = {} self.packetouts = {} self.packetouts_store = {} self.reputation = defaultdict(int) #record test packet #key is (switch_id, header), value is tp_index self.heaer_tpindex = {} #key is test packet index self.send_tpindex = set() self.recv_tpindex = set() self.send_tpindex_store = [] self.ori_tetspackets_size = len(self.testpackets) self.inc_testpackets_size = 0 #for incremental adding rules self.adding_ruleid_set = set() self.adding_ruleid = list() if Detection.INCREMENTAL_ADDING: self.adding_ruleid_set, self.adding_ruleid, self.testpackets = initialize_adding_rules( self.testpackets, adding_path_number=800, is_random=True, is_shuffle=True) self.adding_ruleid = self.adding_ruleid[:3000] #print len(self.adding_ruleid), 'adding rules' if Detection.DEBUG_MSG: 'Adding-rule id:', self.adding_ruleid #for incremental deleting rules self.deleting_ruleid = list() if Detection.INCREMENTAL_DELETING: self.deleting_ruleid = initialize_deleting_rules( self.testpackets, deleting_path_number=700, is_random=True, is_shuffle=True) self.deleting_ruleid = self.deleting_ruleid[:3000] #print len(self.deleting_ruleid), 'deleting rules' if Detection.DEBUG_MSG: 'Deleting-rule id:', self.deleting_ruleid #simulate the attacker #key is rule id which is compromised by switch self.persistent_fault = initialize_persistent_fault( Detection.PERSISTENT_NUM, self.rules, self.testpackets) self.nonpersistent_fault = initialize_nonpersistent_fault( Detection.NONPERSISTENT_NUM, self.rules, self.testpackets, self.persistent_fault) self.catch_fault = set() if Detection.PERSISTENT_NUM > 0: print >> sys.stderr, 'Presistent fault:', self.persistent_fault #print 'Non-presistent fault:', self.nonpersistent_fault #for recording traffic self.packet_counter = 0 self.traffic = [] self.traffic_start_time = 0.0 self.bucket = TokenBucket(Detection.TRAFFIC_RATE, Detection.TRAFFIC_RATE) #for experiment script #write_controller_status(1) print >> sys.stderr, 'Waiting for switches to connect...'
class Detection(app_manager.RyuApp): OFP_VERSIONS = [ofproto_v1_3.OFP_VERSION] DEBUG_MSG = False TRAFFIC_MSG = False INCREMENTAL_ADDING = False INCREMENTAL_DELETING = False DETOUR = False MONITOR = True INCREMENTAL_INDEX = 50000 PERSISTENT_NUM = 0 NONPERSISTENT_NUM = 0 THRESHOLD = -3 TRAFFIC_RATE = 250 * 1024 #(B/s) PACKET_SIZE = 64 #(Bytes) CONFIG_NAME = 'config/config' FILENAME = '' def __init__(self, *args, **kwargs): super(Detection, self).__init__(*args, **kwargs) random.seed() sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) sys.stdin = os.fdopen(sys.stdin.fileno(), 'r', 0) print >> sys.stderr, 'Loading topology' self.read_config() self.rules = load_rules(Detection.FILENAME + '.rules.pkl') self.topology_graph = load_topology(Detection.FILENAME + '.graphml') self.testpackets = load_testpackets(Detection.FILENAME + '.testpackets.pkl') self.datapaths = {} self.packetouts = {} self.packetouts_store = {} self.reputation = defaultdict(int) #record test packet #key is (switch_id, header), value is tp_index self.heaer_tpindex = {} #key is test packet index self.send_tpindex = set() self.recv_tpindex = set() self.send_tpindex_store = [] self.ori_tetspackets_size = len(self.testpackets) self.inc_testpackets_size = 0 #for incremental adding rules self.adding_ruleid_set = set() self.adding_ruleid = list() if Detection.INCREMENTAL_ADDING: self.adding_ruleid_set, self.adding_ruleid, self.testpackets = initialize_adding_rules( self.testpackets, adding_path_number=800, is_random=True, is_shuffle=True) self.adding_ruleid = self.adding_ruleid[:3000] #print len(self.adding_ruleid), 'adding rules' if Detection.DEBUG_MSG: 'Adding-rule id:', self.adding_ruleid #for incremental deleting rules self.deleting_ruleid = list() if Detection.INCREMENTAL_DELETING: self.deleting_ruleid = initialize_deleting_rules( self.testpackets, deleting_path_number=700, is_random=True, is_shuffle=True) self.deleting_ruleid = self.deleting_ruleid[:3000] #print len(self.deleting_ruleid), 'deleting rules' if Detection.DEBUG_MSG: 'Deleting-rule id:', self.deleting_ruleid #simulate the attacker #key is rule id which is compromised by switch self.persistent_fault = initialize_persistent_fault( Detection.PERSISTENT_NUM, self.rules, self.testpackets) self.nonpersistent_fault = initialize_nonpersistent_fault( Detection.NONPERSISTENT_NUM, self.rules, self.testpackets, self.persistent_fault) self.catch_fault = set() if Detection.PERSISTENT_NUM > 0: print >> sys.stderr, 'Presistent fault:', self.persistent_fault #print 'Non-presistent fault:', self.nonpersistent_fault #for recording traffic self.packet_counter = 0 self.traffic = [] self.traffic_start_time = 0.0 self.bucket = TokenBucket(Detection.TRAFFIC_RATE, Detection.TRAFFIC_RATE) #for experiment script #write_controller_status(1) print >> sys.stderr, 'Waiting for switches to connect...' def read_config(self): with open(Detection.CONFIG_NAME, 'r') as f: for line in f: var, val = line.strip().split('=') if var == 'TOPOLOGY_FILE': Detection.FILENAME += val if var == 'DETECTION_THRESHOLD': Detection.THRESHOLD = 0 - int(val) if var == 'TEST_PACKET_RATE(K/bytes)': Detection.TRAFFIC_RATE = int(val) * 1024 if var == 'SIMULATE_ATTACK': Detection.PERSISTENT_NUM = int(val) if var == 'MONITOR': Detection.MONITOR = (val == 'True') def send_testpackets(self): print >> sys.stderr, ' Send test packets' for tp_index, (dp, out) in self.packetouts.iteritems(): while not self.bucket.consume(Detection.PACKET_SIZE): pass dp.send_msg(out) self.packet_counter += 1 if Detection.TRAFFIC_MSG and timeit.default_timer( ) - self.traffic_start_time >= 10: self.traffic.append( self.packet_counter / (timeit.default_timer() - self.traffic_start_time)) self.packet_counter = 0 self.traffic_start_time = timeit.default_timer() @set_ev_cls(ofp_event.EventOFPPacketIn, MAIN_DISPATCHER) def _packet_in_handler(self, ev): msg = ev.msg datapath = msg.datapath ofproto = datapath.ofproto parser = datapath.ofproto_parser in_port = msg.match['in_port'] dpid = datapath.id pkt = packet.Packet(msg.data) src = pkt.get_protocols(ethernet.ethernet)[0].src dst = pkt.get_protocols(ethernet.ethernet)[0].dst ipv4_src = pkt.get_protocols(ipv4.ipv4)[0].src ipv4_dst = pkt.get_protocols(ipv4.ipv4)[0].dst #print len(msg.data), sys.getsizeof(msg.data), sys.getsizeof(pkt) #self.logger.info("packet in dpid %s, %s, %s, %s, %s, %s", dpid, src, dst, ipv4_src, ipv4_dst, in_port) self.recv_tpindex.add(self.header_tpindex[(to_switchid(dpid), ipv4_src)]) def slice_path(self, tp_index): print >> sys.stderr, " Slice path" #slice path and testpacket middle = len(self.testpackets[tp_index].rule_ids) / 2 if tp_index >= Detection.INCREMENTAL_INDEX: new_tp = TestPacket( self.inc_testpackets_size + Detection.INCREMENTAL_INDEX, self.testpackets[tp_index].prefix, self.testpackets[tp_index].rule_ids[middle:]) self.inc_testpackets_size += 1 else: new_tp = TestPacket(self.ori_tetspackets_size, self.testpackets[tp_index].prefix, self.testpackets[tp_index].rule_ids[middle:]) self.ori_tetspackets_size += 1 new_tp.unique_header = self.testpackets[tp_index].unique_header self.testpackets[tp_index].rule_ids = self.testpackets[ tp_index].rule_ids[:middle] #update testpackets #self.testpackets.append(new_tp) self.testpackets[new_tp.index] = new_tp self.send_tpindex.add(new_tp.index) dp, out = generate_packetout(new_tp, self.datapaths, self.rules) self.packetouts[new_tp.index] = (dp, out) tp = self.testpackets[tp_index] self.header_tpindex[(self.rules[tp.rule_ids[-1]].get_switch_id(), tp.get_header())] = tp.index self.header_tpindex[(self.rules[new_tp.rule_ids[-1]].get_switch_id(), new_tp.get_header())] = new_tp.index #add test flow entry self.rules[tp.rule_ids[-1]].is_path_end = True rule = self.rules[tp.rule_ids[-1]] self.add_test_flow_entry(self.datapaths[to_dpid(rule.get_switch_id())], rule) def fault_localization(self): start_time = timeit.default_timer() self.traffic_start_time = timeit.default_timer() print >> sys.stderr, 'Start fault localization at', str(start_time) #self.send_testpackets() #time.sleep(10) while len( self.catch_fault ) < Detection.PERSISTENT_NUM + Detection.NONPERSISTENT_NUM or Detection.TRAFFIC_MSG or Detection.MONITOR: if Detection.TRAFFIC_MSG and timeit.default_timer( ) - start_time >= 110: write_traffic(Detection.FILENAME_PREFIX, self.traffic) break self.send_testpackets() if Detection.TRAFFIC_MSG: continue time.sleep(0.3) #print "send tp index:", self.send_tpindex #print "recv tp index:", self.recv_tpindex suspected_tpindex = self.send_tpindex.difference(self.recv_tpindex) remove_tp_index = [key for key in self.packetouts] for i in remove_tp_index: if i not in suspected_tpindex: self.packetouts_store[i] = self.packetouts[i] self.send_tpindex_store.append(i) del self.packetouts[i] self.send_tpindex.remove(i) for i in suspected_tpindex: tp = self.testpackets[i] for rule_id in tp.rule_ids: self.reputation[rule_id] -= 1 if len(tp.rule_ids) > 1: self.slice_path(i) elif self.reputation[tp.rule_ids[0]] <= Detection.THRESHOLD: if Detection.DETOUR and tp.rule_ids[ 0] in self.detour_pair and self.detour_pair[ tp.rule_ids[0]] != -1: rand = random.randint(1, 10000) self.detour_pair[tp.rule_ids[0]] = -1 Detection.PERSISTENT_NUM -= 1 if rand < 7500: continue else: if Detection.DETOUR: current_time = timeit.default_timer() - start_time fnr = len(self.persistent_fault - self.catch_fault) / float( len(self.persistent_fault)) self.detour_record.append((current_time, fnr)) print >> sys.stderr, 'Detect fault:', tp.rule_ids[0] self.catch_fault.add(tp.rule_ids[0]) if len(suspected_tpindex) == 0: for i in self.send_tpindex_store: self.send_tpindex.add(i) self.packetouts[i] = self.packetouts_store[i] self.send_tpindex_store = [] self.packetouts_store = {} #self.reputation[i] -= 1 #if self.reputation[i] <= Detection.THRESHOLD: # if len(tp.rule_ids) == 1: # print 'Detect fault:', tp.rule_ids[0] # self.catch_fault.add(tp.rule_ids[0]) # else: # self.reputation[i] = 0 # self.slice_path(i) self.recv_tpindex.clear() #time.sleep(0.1) print >> sys.stderr, "End fault localization" print >> sys.stderr, " persistent fault", self.persistent_fault print >> sys.stderr, " non-persistent fault", self.nonpersistent_fault print >> sys.stderr, " catch fault", self.catch_fault end_time = timeit.default_timer() detection_delay = end_time - start_time #print >> sys.stderr, "End time", end_time print >> sys.stderr, "Detection delay", detection_delay #if Detection.PERSISTENT_NUM > 0 and Detection.NONPERSISTENT_NUM == 0: # write_detection_delay(Detection.FILENAME_PREFIX, Detection.PERSISTENT_NUM, detection_delay, self.persistent_fault, Detection.TRAFFIC_RATE, self.packet_counter, Detection.THRESHOLD) #elif Detection.PERSISTENT_NUM == 0 and Detection.NONPERSISTENT_NUM > 0: # write_detection_delay(Detection.FILENAME_PREFIX, Detection.NONPERSISTENT_NUM, detection_delay, self.nonpersistent_fault, Detection.TRAFFIC_RATE, self.packet_counter, Detection.THRESHOLD, prob=Detection.NONPERSISTENT_PROB) #beep() #write_controller_status(0) if (Detection.PERSISTENT_NUM > 0 or Detection.NONPERSISTENT_NUM > 0) and not Detection.DETOUR: os._exit(1) #@set_ev_cls(ofp_event.EventOFPSwitchFeatures, MAIN_DISPATCHER) @set_ev_cls(ofp_event.EventOFPSwitchFeatures, CONFIG_DISPATCHER) def switch_features_handler(self, ev): datapath = ev.msg.datapath self.datapaths[int(datapath.id)] = datapath if len(self.datapaths) == self.topology_graph.num_vertices(): for dpid, dp in self.datapaths.iteritems(): self.initialize_switch(dp) time.sleep(0.2) self.packetouts, self.send_tpindex, self.header_tpindex = initialize_testpackets( self.testpackets, self.datapaths, self.rules) self.ori_packetouts = self.packetouts self.ori_send_tp_index = copy.deepcopy(self.send_tpindex) #self.send_testpackets() #self.fault_localization() #time.sleep(10) fault_localize_thread = hub.spawn(self.fault_localization) attacker_thread = hub.spawn(self.simulate_nonpersistent_fault) if Detection.INCREMENTAL_ADDING: incremental_adding_thread = hub.spawn( self.simulate_incremental_adding) if Detection.INCREMENTAL_DELETING: incremental_deleting_thread = hub.spawn( self.simulate_incremental_deleting) #if Detection.DETOUR: # detour_thread = hub.spawn(self.FN) #ofproto = datapath.ofproto #parser = datapath.ofproto_parser #match = parser.OFPMatch(in_port=78, eth_type=0x0800) #actions = [parser.OFPActionOutput(ofproto.OFPP_CONTROLLER, ofproto.OFPCML_NO_BUFFER)] #self.add_flow(datapath, 0, match, actions) return #Test # # #if datapath.id == to_dpid(72): # match = parser.OFPMatch(in_port=to_dpid(48), eth_type=0x0800, ipv4_src=('179.60.196.0', '255.255.255.0')) # #actions = [parser.OFPActionOutput(73, ofproto.OFPCML_NO_BUFFER)] # actions = [parser.OFPActionOutput(ofproto.OFPP_CONTROLLER, ofproto.OFPCML_NO_BUFFER)] # self.add_flow(datapath, 32768, match, actions) #if len(self.datapaths) == 10: # time.sleep(1) # #self.test_thread = hub.spawn(self._test) # p = packet.Packet() # p.add_protocol(ethernet.ethernet(ethertype=0x800)) # p.add_protocol(ipv4.ipv4(src='179.60.196.0')) # p.add_protocol(tcp.tcp(src_port=1000, dst_port=2000)) # p.serialize() # parser = self.datapaths[to_dpid(72)].ofproto_parser # ofproto = self.datapaths[to_dpid(72)].ofproto # actions = [parser.OFPActionOutput(ofproto.OFPP_TABLE)] # out = parser.OFPPacketOut(datapath=self.datapaths[to_dpid(72)], in_port=to_dpid(48), # buffer_id=self.datapaths[to_dpid(72)].ofproto.OFP_NO_BUFFER, actions=actions, data=p.data) # #out = parser.OFPPacketOut(datapath=datapath, in_port=ofproto.OFPP_CONTROLLER) # for i in xrange(1): # self.datapaths[to_dpid(72)].send_msg(out) def initialize_switch(self, datapath): print >> sys.stderr, '---Initialize switch id:', to_switchid( datapath.id), 'datapath id:', datapath.id switch_id = to_switchid(datapath.id) switch = self.topology_graph.get_switch(switch_id) ofproto = datapath.ofproto parser = datapath.ofproto_parser for rule_id, rule in switch.get_flow_table(table_id=0).iteritems(): #Ignore the rule to simulate persistent attack if rule_id in self.persistent_fault: continue #Ignore adding rule if rule_id in self.adding_ruleid_set: continue out_port = to_dpid(rule.get_out_port()) priority = rule.get_priority() match = self.get_match(parser, rule) actions = [ parser.OFPActionOutput(out_port, ofproto.OFPCML_NO_BUFFER) ] self.add_flow(datapath, priority, match, actions) if rule.is_path_end: self.add_test_flow_entry(datapath, rule) #if rule.is_path_start: # self.start_path_rules[rule.get_path_index()] = rule def add_test_flow_entry(self, datapath, rule): if rule.get_id() in self.persistent_fault: return ofproto = datapath.ofproto parser = datapath.ofproto_parser output_rule = copy.deepcopy(rule) input_rule = copy.deepcopy(rule) sendback_rule = copy.deepcopy(rule) output_rule.set_table_id(output_rule.get_table_id() + 1) output_rule.is_modified_output = True input_rule.set_inst_actions(Rule.GOTO_TABLE, [Rule.OUTPUT]) input_rule.is_modified_input = True sendback_rule.set_table_id(sendback_rule.get_table_id() + 1) sendback_rule.set_prefix( self.testpackets[sendback_rule.get_path_index()].get_prefix()) sendback_rule.set_out_port(ofproto.OFPP_CONTROLLER) sendback_rule.set_priority(32768) sendback_rule.is_sendback = True self.network_sync_insert_rule(output_rule) self.network_sync_insert_rule(input_rule) self.network_sync_insert_rule(sendback_rule) #del original rule self.del_flow(datapath, rule.get_priority(), self.get_match(parser, rule), rule.get_table_id()) #add new input goto table rule #self.add_flow(datapath, input_rule.get_priority(), self.get_match(parser, input_rule), [parser.OFPInstructionGotoTable(input_rule.get_table_id()+1)]) inst = [parser.OFPInstructionGotoTable(output_rule.get_table_id())] mod = parser.OFPFlowMod(datapath=datapath, priority=input_rule.get_priority(), match=self.get_match(parser, input_rule), instructions=inst, table_id=input_rule.get_table_id()) datapath.send_msg(mod) #add new output rule self.add_flow( datapath, output_rule.get_priority(), self.get_match(parser, output_rule), [ parser.OFPActionOutput(to_dpid(output_rule.get_out_port()), ofproto.OFPCML_NO_BUFFER) ], table_id=output_rule.get_table_id()) #add new send back to control rule in_port = to_dpid(sendback_rule.get_in_port()) if sendback_rule.get_prefix().find('/') == -1: ipv4_src = (sendback_rule.get_prefix(), '255.255.255.0') else: ipv4_src = (sendback_rule.get_prefix() [:sendback_rule.get_prefix().index('/')], '255.255.255.0') tcp_src = self.testpackets[ sendback_rule.get_path_index()].get_unique_header() match = parser.OFPMatch(in_port=in_port, eth_type=0x800, ipv4_src=ipv4_src, ip_proto=inet.IPPROTO_TCP, tcp_src=tcp_src) #match = parser.OFPMatch(in_port=in_port, eth_type=0x8847, ipv4_src=ipv4_src, ip_proto=inet.IPPROTO_TCP, mpls_label=tcp_src) #vlan_vid = (0x1000 | self.testpackets[sendback_rule.get_path_index()].get_unique_header()) #match = parser.OFPMatch(in_port=in_port, eth_type=0x800, ipv4_src=ipv4_src, ip_proto=inet.IPPROTO_VRRP, vlan_vid=vlan_vid) self.add_flow(datapath, sendback_rule.get_priority(), match, [ parser.OFPActionOutput(sendback_rule.get_out_port(), ofproto.OFPCML_NO_BUFFER) ], table_id=sendback_rule.get_table_id()) #def del_test_flow_entry(self, rule): # self.network_delete_rule(self.topology_graph.get_switch(rule.get_switch_id()).modified_input_rules[rule.id]) # self.network_delete_rule(self.topology_graph.get_switch(rule.get_switch_id()).modified_output_rules[rule.id]) # self.network_delete_rule(self.topology_graph.get_switch(rule.get_switch_id()).sendback_rules[rule.id]) def get_match(self, parser, rule): in_port = to_dpid(rule.get_in_port()) ##### if rule.get_prefix()[-2:] == '32': ipv4_src = (rule.get_prefix()[:rule.get_prefix().index('/')], '255.255.255.255') elif rule.get_prefix().find('/') == -1: ipv4_src = (rule.get_prefix(), '255.255.255.0') else: ipv4_src = (rule.get_prefix()[:rule.get_prefix().index('/')], '255.255.255.0') return parser.OFPMatch(in_port=in_port, eth_type=0x800, ipv4_src=ipv4_src) #tcp_src = self.testpackets[rule.get_path_index()].get_unique_header() #if not rule.is_incremental: # match = parser.OFPMatch(in_port=in_port, eth_type=0x800, ipv4_src=ipv4_src) #else: # match = parser.OFPMatch(in_port=in_port, eth_type=0x800, ipv4_src=ipv4_src, ip_proto=inet.IPPROTO_TCP, tcp_src=tcp_src) #return match def add_flow(self, datapath, priority, match, actions, table_id=0): ofproto = datapath.ofproto parser = datapath.ofproto_parser inst = [ parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions) ] mod = parser.OFPFlowMod(datapath=datapath, priority=priority, match=match, instructions=inst, table_id=table_id) datapath.send_msg(mod) def del_flow(self, datapath, priority, match, table_id=0): ofproto = datapath.ofproto parser = datapath.ofproto_parser mod = parser.OFPFlowMod(datapath=datapath, priority=priority, match=match, command=ofproto.OFPFC_DELETE, table_id=table_id, out_port=ofproto.OFPP_ANY, out_group=ofproto.OFPG_ANY) datapath.send_msg(mod) def network_insert_rule(self, rule): self.network_sync_insert_rule(rule) datapath = self.datapaths[to_dpid(rule.get_switch_id())] ofproto = datapath.ofproto parser = datapath.ofproto_parser out_port = to_dpid(rule.get_out_port()) priority = rule.get_priority() match = self.get_match(parser, rule) actions = [parser.OFPActionOutput(out_port, ofproto.OFPCML_NO_BUFFER)] if Detection.DEBUG_MSG: print "Insert ", rule, rule.get_table_id() self.add_flow(datapath, priority, match, actions) def network_delete_rule(self, rule): self.network_sync_delete_rule(rule) datapath = self.datapaths[to_dpid(rule.get_switch_id())] parser = datapath.ofproto_parser if Detection.DEBUG_MSG: print "Delete ", rule, rule.get_table_id() self.del_flow(datapath, rule.get_priority(), self.get_match(parser, rule), table_id=rule.get_table_id()) def network_sync_insert_rule(self, rule): if rule.is_incremental: self.topology_graph.get_switch( rule.get_switch_id()).add_incremental_rule(rule) if rule.is_sendback: self.topology_graph.get_switch( rule.get_switch_id()).add_sendback_rule(rule) if rule.is_modified_input: self.topology_graph.get_switch( rule.get_switch_id()).add_modified_input_rule(rule) if rule.is_modified_output: self.topology_graph.get_switch( rule.get_switch_id()).add_modified_output_rule(rule) if rule.is_deleted: self.topology_graph.get_switch( rule.get_switch_id()).add_deleted_rule(rule) def network_sync_delete_rule(self, rule): switch = self.topology_graph.get_switch(rule.get_switch_id()) if rule.is_incremental and rule.id in switch.incremental_rules: del self.topology_graph.get_switch( rule.get_switch_id()).incremental_rules[rule.id] if rule.is_sendback: del self.topology_graph.get_switch( rule.get_switch_id()).sendback_rules[rule.id] if rule.is_modified_input: del self.topology_graph.get_switch( rule.get_switch_id()).modified_input_rules[rule.id] if rule.is_modified_output: del self.topology_graph.get_switch( rule.get_switch_id()).modified_output_rules[rule.id] #if rule.is_deleted: # del self.topology_graph.get_switch(rule.get_switch_id()).deleted_rules[rule.id] def simulate_nonpersistent_fault(self): while len(self.catch_fault) < Detection.NONPERSISTENT_NUM: for rule_id in self.nonpersistent_fault: rule = self.rules[rule_id] dpid = to_dpid(rule.get_switch_id()) datapath = self.datapaths[dpid] ofproto = datapath.ofproto parser = datapath.ofproto_parser #good or bad inst = [] prob = random.uniform(0, 1) if prob > Detection.NONPERSISTENT_PROB: if rule.is_path_end: inst = [ parser.OFPInstructionGotoTable( rule.get_table_id() + 1) ] else: actions = [ parser.OFPActionOutput( to_dpid(rule.get_out_port()), ofproto.OFPCML_NO_BUFFER) ] inst = [ parser.OFPInstructionActions( ofproto.OFPIT_APPLY_ACTIONS, actions) ] mod = parser.OFPFlowMod(datapath=datapath, priority=rule.get_priority(), match=self.get_match(parser, rule), command=ofproto.OFPFC_ADD, \ instructions=inst, table_id=rule.get_table_id()) datapath.send_msg(mod) #print rule.switch_id, prob, rule.is_path_end, inst time.sleep(1) def simulate_incremental_adding(self): delays = [] for rule_id in self.adding_ruleid: rule = self.rules[rule_id] start_time = timeit.default_timer() self.simulate_adding_rule(rule) #time.sleep(1) end_time = timeit.default_timer() delays.append(end_time - start_time) #print "Average delay of adding one rule", (end_time - start_time) / float(len(self.adding_ruleid)) #print "PacketOut number", len(self.packetouts) #write_adding_delay(Detection.FILENAME_PREFIX, delays) print 'Finish adding rules' os._exit(1) def simulate_adding_rule(self, rule): start_tp = None end_tp = None for i in xrange(self.inc_testpackets_size): if i + Detection.INCREMENTAL_INDEX not in self.testpackets: continue inc_tp = self.testpackets[i + Detection.INCREMENTAL_INDEX] start_rule = self.rules[inc_tp.rule_ids[0]] end_rule = self.rules[inc_tp.rule_ids[-1]] if rule.get_out_port() == start_rule.get_switch_id( ) and rule.get_prefix() == start_rule.get_prefix(): start_tp = inc_tp if rule.get_in_port() == end_rule.get_switch_id( ) and rule.get_prefix() == end_rule.get_prefix(): end_tp = inc_tp if Detection.DEBUG_MSG: print rule rule.is_incremental = True r_start = self.rules[ start_tp.rule_ids[0]] if start_tp != None else None r_end = self.rules[end_tp.rule_ids[-1]] if end_tp != None else None #print 'Finish adding rules' if start_tp is not None and end_tp is not None: if Detection.DEBUG_MSG: print '--------------------------1' rule.path_index = start_tp.index self.testpackets[start_tp.index].rule_ids = end_tp.rule_ids + [ rule.id ] + start_tp.rule_ids # end_tp header = intersection(end_tp, rule, start_tp) self.network_delete_rule( self.topology_graph.get_switch( r_end.get_switch_id()).modified_input_rules[r_end.id]) self.network_delete_rule( self.topology_graph.get_switch( r_end.get_switch_id()).modified_output_rules[r_end.id]) self.network_delete_rule( self.topology_graph.get_switch( r_end.get_switch_id()).sendback_rules[r_end.id]) self.network_insert_rule(r_end) self.network_insert_rule(rule) #self.packetouts[index] should change the packet header self.packetouts[start_tp.index] = generate_packetout( self.testpackets[start_tp.index], self.datapaths, self.rules) for rule_id in end_tp.rule_ids: self.rules[rule_id].path_index = start_tp.index del self.packetouts[end_tp.index] del self.testpackets[end_tp.index] self.send_tpindex.remove(end_tp.index) elif start_tp is not None: if Detection.DEBUG_MSG: print '--------------------------2' rule.path_index = start_tp.index self.testpackets[start_tp.index].rule_ids = [rule.id ] + start_tp.rule_ids self.network_insert_rule(rule) # start_tp header = intersection(rule, start_tp) self.packetouts[start_tp.index] = generate_packetout( self.testpackets[start_tp.index], self.datapaths, self.rules) #self.packetouts[index] should change the packet header elif end_tp is not None: if Detection.DEBUG_MSG: print '--------------------------3' rule.path_index = end_tp.index self.testpackets[ end_tp.index].rule_ids = end_tp.rule_ids + [rule.id] # end_tp header = intersection(end_tp, rule) self.network_delete_rule( self.topology_graph.get_switch( r_end.get_switch_id()).modified_input_rules[r_end.id]) self.network_delete_rule( self.topology_graph.get_switch( r_end.get_switch_id()).modified_output_rules[r_end.id]) self.network_delete_rule( self.topology_graph.get_switch( r_end.get_switch_id()).sendback_rules[r_end.id]) self.network_insert_rule(r_end) self.add_test_flow_entry( self.datapaths[to_dpid(rule.get_switch_id())], rule) #self.packetouts[index] should change the packet header self.header_tpindex[(rule.get_switch_id(), end_tp.get_header())] = end_tp.index else: if Detection.DEBUG_MSG: print '--------------------------4' index = self.inc_testpackets_size + Detection.INCREMENTAL_INDEX rule.path_index = index self.testpackets[index] = TestPacket(index, rule.get_prefix(), [rule.id]) self.inc_testpackets_size += 1 self.add_test_flow_entry( self.datapaths[to_dpid(rule.get_switch_id())], rule) self.packetouts[index] = generate_packetout( self.testpackets[index], self.datapaths, self.rules) self.header_tpindex[(rule.get_switch_id(), self.testpackets[index].get_header())] = index self.send_tpindex.add(index) def simulate_incremental_deleting(self): delays = [] for rule_id in self.deleting_ruleid: rule = self.rules[rule_id] start_time = timeit.default_timer() self.simulate_deleting_rule(rule) #time.sleep(1) end_time = timeit.default_timer() delays.append(end_time - start_time) #print "Average delay of deleting one rule", (end_time - start_time) / float(len(self.deleting_ruleid)) #print "PacketOut number", len(self.packetouts) #write_deleting_delay(Detection.FILENAME_PREFIX, delays) print 'Finish deleting rules' os._exit(1) def simulate_deleting_rule(self, rule): tp_index = rule.get_path_index() rule.is_deleted = True self.rules[rule.id].is_deleted = True deleted_number = 0 for i, rule_id in enumerate(self.testpackets[tp_index].rule_ids): if self.rules[rule_id].is_deleted: deleted_number += 1 if len(self.testpackets[tp_index].rule_ids) - deleted_number == 0: self.network_delete_rule(rule) del self.testpackets[tp_index] del self.packetouts[tp_index] self.send_tpindex.remove(tp_index) return for i, rule_id in enumerate(self.testpackets[tp_index].rule_ids): if i == 0 and rule_id == rule.id: if Detection.DEBUG_MSG: print '--------------------------1' self.network_delete_rule(rule) self.testpackets[tp_index].rule_ids = self.testpackets[ tp_index].rule_ids[1:] self.packetouts[tp_index] = generate_packetout( self.testpackets[tp_index], self.datapaths, self.rules) end_rule = self.rules[self.testpackets[tp_index].rule_ids[-1]] self.header_tpindex[( end_rule.get_switch_id(), self.testpackets[tp_index].get_header())] = tp_index return elif i == len(self.testpackets[tp_index].rule_ids ) - 1 and rule_id == rule.id: if Detection.DEBUG_MSG: print '--------------------------2' self.network_delete_rule( self.topology_graph.get_switch( rule.get_switch_id()).modified_input_rules[rule.id]) self.network_delete_rule( self.topology_graph.get_switch( rule.get_switch_id()).modified_output_rules[rule.id]) self.network_delete_rule( self.topology_graph.get_switch( rule.get_switch_id()).sendback_rules[rule.id]) self.testpackets[tp_index].rule_ids = self.testpackets[ tp_index].rule_ids[:-1] end_rule = self.rules[self.testpackets[tp_index].rule_ids[-1]] self.header_tpindex[( end_rule.get_switch_id(), self.testpackets[tp_index].get_header())] = tp_index self.add_test_flow_entry( self.datapaths[to_dpid(end_rule.get_switch_id())], end_rule) return if rule_id == rule.id: if Detection.DEBUG_MSG: print '--------------------------3' self.network_delete_rule(rule) datapath = self.datapaths[to_dpid(rule.get_switch_id())] ofproto = datapath.ofproto parser = datapath.ofproto_parser in_port = to_dpid(rule.get_in_port()) ipv4_src = (rule.get_prefix()[:rule.get_prefix().index('/')], '255.255.255.0') tcp_src = self.testpackets[tp_index].get_unique_header() match = parser.OFPMatch(in_port=in_port, eth_type=0x800, ipv4_src=ipv4_src, ip_proto=inet.IPPROTO_TCP, tcp_src=tcp_src) self.add_flow( datapath, rule.get_priority(), match, [ parser.OFPActionOutput(to_dpid(rule.get_out_port()), ofproto.OFPCML_NO_BUFFER) ], table_id=rule.get_table_id()) if Detection.DEBUG_MSG: print "Insert ", rule, rule.get_table_id() return