def init_pieces(self): blocks_per_piece = int(math.ceil(self.piece_length / self.block_size)) start_piece = math.floor(self.torrent.media_file.start_byte / self.piece_length) end_piece = math.ceil(self.torrent.media_file.end_byte / self.piece_length) current_byte = start_piece * self.piece_length start_time = current_time() for index in range(end_piece - start_piece): piece_index = start_piece + index if current_byte + self.piece_length > self.torrent.total_size: # last piece, is not full length self._pieces[piece_index] = Piece( self, piece_index, piece_index * blocks_per_piece, current_byte, self.torrent.total_size - current_byte) else: self._pieces[piece_index] = Piece( self, piece_index, piece_index * blocks_per_piece, current_byte, self.piece_length) current_byte += self.piece_length self.last_piece_index = piece_index self.init_done = True Logger().write( LogVerbosity.Info, "Pieces initialized, " + str(len(self._pieces)) + " pieces created in " + str(current_time() - start_time) + "ms")
def tick(self): tick_time = current_time() cur_list = list(self.work_items) for i in range(len(cur_list)): work_item = cur_list[i] if work_item.last_run_time + work_item.interval < tick_time: Timing().start_timing("Engine item " + work_item.name) self.current_item = work_item self.current_item_log = work_item.name self.start_time = current_time() if not self.running: return result = work_item.action() self.current_item = None self.current_item_log = "" test_time = current_time() work_item.last_run_time = test_time work_item.last_run_duration = test_time - self.start_time if work_item.interval == -1: self.work_items.remove(work_item) elif not result: self.work_items.remove(work_item) if work_item.name not in self.timing: self.timing[work_item.name] = TimingObject(work_item.name) self.timing[work_item.name].add_time(current_time() - self.start_time) Timing().stop_timing("Engine item " + work_item.name) sleep(0) self.own_time.add_time(current_time() - tick_time)
def wait_for(max_time, action): start = current_time() while current_time() - start < max_time: if action(): return True time.sleep(0.5) return False
def update_timeout(self): if self.peer is None or self.peer.state != PeerState.Started: return True canceled = 0 timed_out_blocks = [ (block, request_time, timed_out) for block, request_time, timed_out in self.downloading if current_time() - request_time > self.get_priority_timeout( self.peer.torrent.data_manager._pieces[ block.piece_index].priority) and not timed_out ] for block_request in timed_out_blocks: block = block_request[0] self.downloading.remove(block_request) self.downloading.append((block_request[0], block_request[1], True)) self.downloading_log = ", ".join( [str(x[0].index) for x in self.downloading]) # cancel_msg = CancelMessage(block.piece_index, block.start_byte_in_piece, block.length) # self.peer.protocol_logger.update("Sending cancel (timeout)") # Logger().write(LogVerbosity.All, str(self.peer.id) + ' Sending cancel for piece ' + str(block.piece_index) + ", block " + str(cancel_msg.offset // 16384)) # self.peer.connection_manager.send(cancel_msg.to_bytes()) block.remove_downloader(self.peer) canceled += 1 if canceled: Logger().write( LogVerbosity.Debug, str(self.peer.id) + " canceled " + str(canceled) + " blocks") self.timed_out_blocks = current_time()
def __check_update(self): self.__running = True while self.__running: self.__wait_event.wait(self.__update_interval) self.__wait_event.clear() if current_time() - self.__last_update < (self.__update_interval * 1000): continue if self.__changed: self.__changed = False self.__last_update = current_time() dic = { k: v for k, v in self.__dict__.items() if not k.startswith("_") } last_update = namedtuple("Observable", dic.keys())(*dic.values()) for cb in self.__callbacks: try: cb(self.__last_update_state or self, self) except Exception as e: Logger().write_error(e, "Exception in observer callback") self.__last_update_state = last_update
def set_device_state(self, home): if home: self.last_seen = current_time() self.last_home_state = True if not self.home_state: self.home_state = True Logger().write(LogVerbosity.Info, "Device " + self.name + " came home") return if not home and not self.home_state: # still not home return if self.last_home_state: # Now not home, last state was home. Start timing self.last_home_state = False self.timeout_start_time = current_time() return else: # Now not home, last state was also not home. Check timeout if current_time( ) - self.timeout_start_time > self.gone_interval * 1000: self.home_state = False Logger().write(LogVerbosity.Info, "Device " + self.name + " left home")
def requeue(self, start_position): Logger().write(LogVerbosity.All, "Starting download queue queuing") start_time = current_time() left = 0 self.queue.clear() for piece in self.torrent.data_manager.get_pieces_after_index( start_position): if piece.written: continue if piece.start_byte > self.torrent.media_file.end_byte or piece.end_byte < self.torrent.media_file.start_byte: continue piece.reset() self.queue.append(piece) left += piece.length first = "none" length = len(self.queue) if length > 0: first = str(self.queue[0].index) self.queue_log = "length: " + str(length) + ", first: " + first self.slow_peer_piece_offset = 15000000 // self.torrent.data_manager.piece_length # TODO Setting Logger().write( LogVerbosity.Debug, "Queueing took " + str(current_time() - start_time) + "ms for " + str(len(self.queue)) + " items") self.torrent.left = left if self.torrent.left > 0 and self.torrent.state == TorrentState.Done: self.torrent.restart_downloading() self.update_priority(True)
def handle_write(self): success = self.connection.send(self.to_send_bytes) self.to_send_bytes.clear() self.buffer_out_size = 0 self.last_send = current_time() self._last_communication = current_time() if not success: self.peer.stop_async("Write error")
def value(self): if current_time() - self._last_get < 100: return self.parent.speed_log self.items = [x for x in self.items if current_time() - x[0] < self._retaining] last_result = sum([x[1] for x in self.items]) // self.seconds_average self.parent.speed_log = last_result self._last_get = current_time() return last_result
def get_token(node, bytes): if current_time() - TokenManager.last_generated > 5 * 60 * 1000: TokenManager.last_generated = current_time() TokenManager.previous_secret = TokenManager.secret TokenManager.secret = os.urandom(10) copy = bytearray(bytes) node_bytes = node.ip_port_bytes() for i in range(len(node_bytes)): copy[i] |= node_bytes[i] return hashlib.sha1(copy).digest()
def update_priority(self, full=False): if not self.init: return True if self.torrent.state == TorrentState.Done: if self.queue: self.queue = [] self.queue_log = "" return True start_time = current_time() amount = 100 if full: amount = len(self.torrent.data_manager._pieces) start = self.torrent.stream_position if full: start = self.torrent.media_file.start_piece( self.torrent.piece_length) pieces_to_look_at = self.torrent.data_manager.get_pieces_by_index_range( start, start + amount) for piece in pieces_to_look_at: piece.priority = self.prioritize_piece_index(piece.index) if piece.priority > 100 and piece.max_priority_set_time == 0: piece.max_priority_set_time = current_time() else: piece.max_priority_set_time = 0 if full: self.queue = sorted(self.queue, key=lambda x: x.priority, reverse=True) first = "none" length = len(self.queue) if length > 0: first = str(self.queue[0].index) self.queue_log = "length: " + str(length) + ", first: " + first if self.queue: piece = self.queue[0] Logger().write( LogVerbosity.Debug, "Highest prio: " + str(piece.index) + ": " + str(piece.priority) + "%, " + str(len([x for x in piece.blocks.values() if not x.done])) + " block left, took " + str(current_time() - start_time) + "ms, full: " + str(full)) else: Logger().write(LogVerbosity.Debug, "No prio: queue empty") self.prio = True return True
def process(self): while self.running: start_time = current_time() peers_to_process = self.torrent.peer_manager.connected_peers Timing().start_timing("peer_processing") for peer in peers_to_process: peer.metadata_manager.update() peer.download_manager.update_timeout() Timing().stop_timing("peer_processing") spend_time = current_time() - start_time time.sleep(0.1 - (spend_time / 1000))
def start_url(self, title, url): self.stop_play() VLCPlayer().play(url, 0) if Settings.get_bool("slave"): self.history_id, = SlaveClientController.request_master( "add_watched_url", 5, url, current_time()) else: self.history_id = Database().add_watched_url(url, current_time()) self.media_data.start_update() self.media_data.type = "Url" self.media_data.title = title self.media_data.stop_update() TVManager().switch_input_to_pi()
def start_observing(self): Logger().write(LogVerbosity.Debug, "Start observing light data") self.observing = True if self.observing_end > current_time(): Logger().write(LogVerbosity.All, "Still observing, not starting again") return # still observing, the check observing thread will renew if not self.check_state(): return self.observing_end = current_time() + 30000 groups_commands = self.api(self.gateway.get_groups()) result = self.api(groups_commands) for group in result: self.observe_group(group)
def update_stat(self, key, value): Logger().write(LogVerbosity.All, "Database update stat " + str(key)) database, cursor = self.connect() cursor.execute("SELECT * FROM Stats WHERE Name=?", [key]) data = cursor.fetchall() if not data: cursor.execute( "INSERT INTO Stats (Name, Val, LastUpdate) VALUES (?, ?, ?)", [key, value, current_time()]) else: cursor.execute("UPDATE Stats SET Val=?, LastUpdate=? WHERE Name=?", [value, current_time(), key]) database.commit() database.close()
def refresh_session_key(self, client_key, new_session_key, ip, user_agent): if self.slave: raise PermissionError("Cant call refresh_session_key on slave") Logger().write(LogVerbosity.All, "Database refresh session key") database, cursor = self.connect() cursor.execute( "UPDATE Clients SET SessionKey=?, LastSeen=? WHERE ClientKey=?", [new_session_key, current_time(), client_key]) cursor.execute( "INSERT INTO AuthorizeAttempts (ClientKey, IP, UserAgent, Timestamp, Type, Successful) VALUES (?, ?, ?, ?, ?, ?)", [client_key, ip, user_agent, current_time(), "Refresh", True]) database.commit() database.close()
def on_init(client_name, key): Logger().write(LogVerbosity.Info, "Init slave: " + client_name) if key != SecureSettings.get_string("master_key"): Logger().write(LogVerbosity.Info, "Slave authentication failed") disconnect() return False slave = APIController.slaves.get_slave(client_name) if slave is not None: if slave.connected: Logger().write( LogVerbosity.Info, "Slave " + str(client_name) + " connected twice?") disconnect() return False else: Logger().write(LogVerbosity.Info, "Slave " + str(client_name) + " reconnected") slave.reconnect(request.sid) APIController.slaves.changed() return True client = WebsocketClient(request.sid, current_time()) client.authenticated = True APIController.slaves.add_slave( SlaveClient(APIController.next_id(), client_name, client)) return client.authenticated
def check(self): for pending in list(self.awaiting_messages): if current_time() - pending.send_at > 10000: Logger().write(LogVerbosity.All, "DHT message timeout") self.node_timeout_handler(pending.message.ip, pending.message.port) pending.on_timeout() self.awaiting_messages.remove(pending) for received in list(self.received_messages): if isinstance(received.message, QueryDHTMessage): self.query_handler(received.ip, received.port, received.message) self.received_messages.remove(received) continue elif isinstance(received.message, ResponseDHTMessage): pending = [ x for x in self.awaiting_messages if x.message.message.transaction_id == received.message.transaction_id ] if len(pending) == 0: Logger().write(LogVerbosity.All, "DHT response for no request (timed out?)") self.received_messages.remove(received) continue Logger().write(LogVerbosity.All, "DHT message response") pending[0].on_response(received.message) # answer to request self.received_messages.remove(received) self.awaiting_messages.remove(pending[0])
def update_peer(self, peer, from_state, to_state): if from_state == PeerState.Starting and to_state == PeerState.Started: self.connecting_peers.remove(peer) self.connected_peers.append(peer) elif from_state == PeerState.Starting and to_state == PeerState.Stopping: self.connecting_peers.remove(peer) self.cant_connect_peers.append(peer) elif from_state == PeerState.Started and to_state == PeerState.Stopping: self.connected_peers.remove(peer) self.disconnected_peers.append( (peer.uri, peer.source, current_time(), peer.counter.total)) self.high_speed_peers = len([ x for x in self.connected_peers if x.peer_speed == PeerSpeed.High ]) self.medium_speed_peers = len([ x for x in self.connected_peers if x.peer_speed == PeerSpeed.Medium ]) self.connected_peers = sorted(self.connected_peers, key=lambda x: x.counter.value, reverse=True) self.disconnected_peers = sorted(self.disconnected_peers, key=lambda x: x[3], reverse=True) self.connecting_peers_log = len(self.connecting_peers) self.connected_peers_log = len(self.connected_peers) self.cant_connect_peers_log = len(self.cant_connect_peers) self.disconnected_peers_log = len(self.disconnected_peers)
def stop_slowest_peer(self): if not self.should_stop_peers(): return True if self.max_peers_connected - len(self.connected_peers) > 3: return True # No need to stop slowest if we have enough room to connect more peers_to_check = [ x for x in self.connected_peers if current_time() - x.connection_manager.connected_on > 30000 ] peers_to_check = sorted(peers_to_check, key=lambda x: x.counter.value) if len(peers_to_check) == 0: return True slowest = peers_to_check[0] if slowest.counter.value > 100000: # if the slowest peer is downloading with more than 100kbps we're fine return True Logger().write( LogVerbosity.Info, str(slowest.id) + " stopping peer to find a potential faster one. Peer speed last 5 seconds was " + str(write_size(slowest.counter.value)) + ", total: " + str(write_size(slowest.counter.total))) slowest.stop_async("Slowest") return True
def update_node(self, id): for bucket in self.buckets: for node in bucket.nodes: if node.byte_id == id: node.seen() bucket.last_update = current_time() return
def stop(self): run_time = current_time() - self.current_start self.high = max(run_time, self.high) self.low = min(run_time, self.low) self.total_items += 1 self.total_time += run_time self.average = self.total_time / self.total_items
def check_peer_state(self): zero_speed_checks = 0 while self.state == PeerState.Started: if current_time() - self.connection_manager.connected_on < 20000: time.sleep(1) continue if not self.torrent.peer_manager.should_stop_peers(): time.sleep(1) continue if self.counter.total == 0: Logger().write(LogVerbosity.Info, str(self.id) + " stopping not downloading peer") self.stop_async( "Not downloading" ) # Stop since we haven't downloaded anything since connecting return if self.counter.value == 0: zero_speed_checks += 1 if zero_speed_checks >= 10: Logger().write( LogVerbosity.Info, str(self.id) + " stopping currently not downloading peer") self.stop_async( "Not recently downloading" ) # Stop since we haven't downloaded anything in the last 10 seconds return else: zero_speed_checks = 0 time.sleep(1)
def add_thread(self, thread): thread.history_entry = ThreadEntry(thread.thread_name, current_time()) if thread.thread_name not in self.thread_history: self.thread_history[thread.thread_name] = [] self.thread_history[thread.thread_name].append(thread.history_entry) self.threads.append(thread) self.thread_count = len(self.threads)
def announce_torrent(self, torrent): self.last_announce = current_time() announce_message = TrackerMessages.TrackerAnnounceMessage.for_http(torrent.info_hash, 2, torrent.total_size - torrent.left, torrent.left, torrent.uploaded, self.tracker_peer_request_amount) path = self.uri.path + announce_message.as_param_string() response = RequestFactory.make_request(path) if response is None: return False try: response_dict = Bencode.bdecode(response) except BTFailure: Logger().write(LogVerbosity.Info, 'Invalid tracker response: ' + str(response)) return False if b"peers" not in response_dict: return False peers_data = response_dict[b"peers"] total_peers = int(len(peers_data) / 6) offset = 0 peers = [] for index in range(total_peers): peers.append(uri_from_bytes(peers_data[offset:offset + 6])) offset += 6 EventManager.throw_event(EventType.PeersFound, [peers, PeerSource.HttpTracker])
def execute(self): while self.running: Timing().start_timing("IO") # Select in/outputs input_peers = self.torrent.peer_manager.get_peers_for_reading() received_messages = [] for peer in input_peers: messages_size, messages = peer.connection_manager.handle_read() if messages_size > 0: self.average_download_counter.add_value(messages_size) time = current_time() received_messages += [(peer, x, time) for x in messages] if len(received_messages) != 0: self.torrent.message_processor.process_messages(received_messages) for peer in self.torrent.peer_manager.connected_peers: peer.download_manager.update_requests() output_peers = self.torrent.peer_manager.get_peers_for_writing() for peer in output_peers: peer.connection_manager.handle_write() Timing().stop_timing("IO") sleep(0.005)
def runner(self): while self.running: sleep(0.01) if not self.running: break self.last_tick = current_time() self.tick()
def add_client(self, client_key, session_key, ip, user_agent): if self.slave: raise PermissionError("Cant call add_client on slave") Logger().write(LogVerbosity.Debug, "Database add client") database, cursor = self.connect() time = current_time() cursor.execute( "INSERT INTO Clients (ClientKey, SessionKey, Issued, LastSeen) Values (?, ?, ?, ?)", [client_key, session_key, time, time]) cursor.execute( "INSERT INTO AuthorizeAttempts (ClientKey, IP, UserAgent, Timestamp, Type, Successful) VALUES (?, ?, ?, ?, ?, ?)", [client_key, ip, user_agent, current_time(), "Login", True]) database.commit() database.close()
def _start_playing_torrent(self): if Settings.get_bool("slave"): self.history_id, = SlaveClientController.request_master( "add_watched_torrent", 5, self.media_data.type, self.media_data.title, self.media_data.id, self.torrent.uri, self.torrent.media_file.path, self.media_data.image, self.media_data.season, self.media_data.episode, current_time()) else: self.history_id = Database().add_watched_torrent( self.media_data.type, self.media_data.title, self.media_data.id, self.torrent.uri, self.torrent.media_file.path, self.media_data.image, self.media_data.season, self.media_data.episode, current_time()) VLCPlayer().play("http://localhost:50009/torrent", self.media_data.start_from)
def add_node(self, node): if len(self.nodes) >= self.max_nodes: self.nodes = [ x for x in self.nodes if x.node_state != NodeState.Bad ] self.nodes.append(node) self.last_changed = current_time()