def init_pieces(self):
blocks_per_piece = int(math.ceil(self.piece_length / self.block_size))
start_piece = math.floor(self.torrent.media_file.start_byte /
self.piece_length)
end_piece = math.ceil(self.torrent.media_file.end_byte /
self.piece_length)
current_byte = start_piece * self.piece_length
start_time = current_time()
for index in range(end_piece - start_piece):
piece_index = start_piece + index
if current_byte + self.piece_length > self.torrent.total_size:
# last piece, is not full length
self._pieces[piece_index] = Piece(
self, piece_index, piece_index * blocks_per_piece,
current_byte, self.torrent.total_size - current_byte)
else:
self._pieces[piece_index] = Piece(
self, piece_index, piece_index * blocks_per_piece,
current_byte, self.piece_length)
current_byte += self.piece_length
self.last_piece_index = piece_index
self.init_done = True
Logger().write(
LogVerbosity.Info,
"Pieces initialized, " + str(len(self._pieces)) +
" pieces created in " + str(current_time() - start_time) + "ms")
def tick(self):
tick_time = current_time()
cur_list = list(self.work_items)
for i in range(len(cur_list)):
work_item = cur_list[i]
if work_item.last_run_time + work_item.interval < tick_time:
Timing().start_timing("Engine item " + work_item.name)
self.current_item = work_item
self.current_item_log = work_item.name
self.start_time = current_time()
if not self.running:
return
result = work_item.action()
self.current_item = None
self.current_item_log = ""
test_time = current_time()
work_item.last_run_time = test_time
work_item.last_run_duration = test_time - self.start_time
if work_item.interval == -1:
self.work_items.remove(work_item)
elif not result:
self.work_items.remove(work_item)
if work_item.name not in self.timing:
self.timing[work_item.name] = TimingObject(work_item.name)
self.timing[work_item.name].add_time(current_time() - self.start_time)
Timing().stop_timing("Engine item " + work_item.name)
sleep(0)
self.own_time.add_time(current_time() - tick_time)
def wait_for(max_time, action):
start = current_time()
while current_time() - start < max_time:
if action():
return True
time.sleep(0.5)
return False
def update_timeout(self):
if self.peer is None or self.peer.state != PeerState.Started:
return True
canceled = 0
timed_out_blocks = [
(block, request_time, timed_out)
for block, request_time, timed_out in self.downloading
if current_time() - request_time > self.get_priority_timeout(
self.peer.torrent.data_manager._pieces[
block.piece_index].priority) and not timed_out
]
for block_request in timed_out_blocks:
block = block_request[0]
self.downloading.remove(block_request)
self.downloading.append((block_request[0], block_request[1], True))
self.downloading_log = ", ".join(
[str(x[0].index) for x in self.downloading])
# cancel_msg = CancelMessage(block.piece_index, block.start_byte_in_piece, block.length)
# self.peer.protocol_logger.update("Sending cancel (timeout)")
# Logger().write(LogVerbosity.All, str(self.peer.id) + ' Sending cancel for piece ' + str(block.piece_index) + ", block " + str(cancel_msg.offset // 16384))
# self.peer.connection_manager.send(cancel_msg.to_bytes())
block.remove_downloader(self.peer)
canceled += 1
if canceled:
Logger().write(
LogVerbosity.Debug,
str(self.peer.id) + " canceled " + str(canceled) + " blocks")
self.timed_out_blocks = current_time()
def __check_update(self):
self.__running = True
while self.__running:
self.__wait_event.wait(self.__update_interval)
self.__wait_event.clear()
if current_time() - self.__last_update < (self.__update_interval *
1000):
continue
if self.__changed:
self.__changed = False
self.__last_update = current_time()
dic = {
k: v
for k, v in self.__dict__.items() if not k.startswith("_")
}
last_update = namedtuple("Observable",
dic.keys())(*dic.values())
for cb in self.__callbacks:
try:
cb(self.__last_update_state or self, self)
except Exception as e:
Logger().write_error(e,
"Exception in observer callback")
self.__last_update_state = last_update
def set_device_state(self, home):
if home:
self.last_seen = current_time()
self.last_home_state = True
if not self.home_state:
self.home_state = True
Logger().write(LogVerbosity.Info,
"Device " + self.name + " came home")
return
if not home and not self.home_state:
# still not home
return
if self.last_home_state:
# Now not home, last state was home. Start timing
self.last_home_state = False
self.timeout_start_time = current_time()
return
else:
# Now not home, last state was also not home. Check timeout
if current_time(
) - self.timeout_start_time > self.gone_interval * 1000:
self.home_state = False
Logger().write(LogVerbosity.Info,
"Device " + self.name + " left home")
def requeue(self, start_position):
Logger().write(LogVerbosity.All, "Starting download queue queuing")
start_time = current_time()
left = 0
self.queue.clear()
for piece in self.torrent.data_manager.get_pieces_after_index(
start_position):
if piece.written:
continue
if piece.start_byte > self.torrent.media_file.end_byte or piece.end_byte < self.torrent.media_file.start_byte:
continue
piece.reset()
self.queue.append(piece)
left += piece.length
first = "none"
length = len(self.queue)
if length > 0:
first = str(self.queue[0].index)
self.queue_log = "length: " + str(length) + ", first: " + first
self.slow_peer_piece_offset = 15000000 // self.torrent.data_manager.piece_length # TODO Setting
Logger().write(
LogVerbosity.Debug,
"Queueing took " + str(current_time() - start_time) + "ms for " +
str(len(self.queue)) + " items")
self.torrent.left = left
if self.torrent.left > 0 and self.torrent.state == TorrentState.Done:
self.torrent.restart_downloading()
self.update_priority(True)
def handle_write(self):
success = self.connection.send(self.to_send_bytes)
self.to_send_bytes.clear()
self.buffer_out_size = 0
self.last_send = current_time()
self._last_communication = current_time()
if not success:
self.peer.stop_async("Write error")
def value(self):
if current_time() - self._last_get < 100:
return self.parent.speed_log
self.items = [x for x in self.items if current_time() - x[0] < self._retaining]
last_result = sum([x[1] for x in self.items]) // self.seconds_average
self.parent.speed_log = last_result
self._last_get = current_time()
return last_result
def get_token(node, bytes):
if current_time() - TokenManager.last_generated > 5 * 60 * 1000:
TokenManager.last_generated = current_time()
TokenManager.previous_secret = TokenManager.secret
TokenManager.secret = os.urandom(10)
copy = bytearray(bytes)
node_bytes = node.ip_port_bytes()
for i in range(len(node_bytes)):
copy[i] |= node_bytes[i]
return hashlib.sha1(copy).digest()
def update_priority(self, full=False):
if not self.init:
return True
if self.torrent.state == TorrentState.Done:
if self.queue:
self.queue = []
self.queue_log = ""
return True
start_time = current_time()
amount = 100
if full:
amount = len(self.torrent.data_manager._pieces)
start = self.torrent.stream_position
if full:
start = self.torrent.media_file.start_piece(
self.torrent.piece_length)
pieces_to_look_at = self.torrent.data_manager.get_pieces_by_index_range(
start, start + amount)
for piece in pieces_to_look_at:
piece.priority = self.prioritize_piece_index(piece.index)
if piece.priority > 100 and piece.max_priority_set_time == 0:
piece.max_priority_set_time = current_time()
else:
piece.max_priority_set_time = 0
if full:
self.queue = sorted(self.queue,
key=lambda x: x.priority,
reverse=True)
first = "none"
length = len(self.queue)
if length > 0:
first = str(self.queue[0].index)
self.queue_log = "length: " + str(length) + ", first: " + first
if self.queue:
piece = self.queue[0]
Logger().write(
LogVerbosity.Debug, "Highest prio: " + str(piece.index) +
": " + str(piece.priority) + "%, " +
str(len([x for x in piece.blocks.values() if not x.done])) +
" block left, took " + str(current_time() - start_time) +
"ms, full: " + str(full))
else:
Logger().write(LogVerbosity.Debug, "No prio: queue empty")
self.prio = True
return True
def process(self):
while self.running:
start_time = current_time()
peers_to_process = self.torrent.peer_manager.connected_peers
Timing().start_timing("peer_processing")
for peer in peers_to_process:
peer.metadata_manager.update()
peer.download_manager.update_timeout()
Timing().stop_timing("peer_processing")
spend_time = current_time() - start_time
time.sleep(0.1 - (spend_time / 1000))
def start_url(self, title, url):
self.stop_play()
VLCPlayer().play(url, 0)
if Settings.get_bool("slave"):
self.history_id, = SlaveClientController.request_master(
"add_watched_url", 5, url, current_time())
else:
self.history_id = Database().add_watched_url(url, current_time())
self.media_data.start_update()
self.media_data.type = "Url"
self.media_data.title = title
self.media_data.stop_update()
TVManager().switch_input_to_pi()
def start_observing(self):
Logger().write(LogVerbosity.Debug, "Start observing light data")
self.observing = True
if self.observing_end > current_time():
Logger().write(LogVerbosity.All, "Still observing, not starting again")
return # still observing, the check observing thread will renew
if not self.check_state():
return
self.observing_end = current_time() + 30000
groups_commands = self.api(self.gateway.get_groups())
result = self.api(groups_commands)
for group in result:
self.observe_group(group)
def update_stat(self, key, value):
Logger().write(LogVerbosity.All, "Database update stat " + str(key))
database, cursor = self.connect()
cursor.execute("SELECT * FROM Stats WHERE Name=?", [key])
data = cursor.fetchall()
if not data:
cursor.execute(
"INSERT INTO Stats (Name, Val, LastUpdate) VALUES (?, ?, ?)",
[key, value, current_time()])
else:
cursor.execute("UPDATE Stats SET Val=?, LastUpdate=? WHERE Name=?",
[value, current_time(), key])
database.commit()
database.close()
def refresh_session_key(self, client_key, new_session_key, ip, user_agent):
if self.slave:
raise PermissionError("Cant call refresh_session_key on slave")
Logger().write(LogVerbosity.All, "Database refresh session key")
database, cursor = self.connect()
cursor.execute(
"UPDATE Clients SET SessionKey=?, LastSeen=? WHERE ClientKey=?",
[new_session_key, current_time(), client_key])
cursor.execute(
"INSERT INTO AuthorizeAttempts (ClientKey, IP, UserAgent, Timestamp, Type, Successful) VALUES (?, ?, ?, ?, ?, ?)",
[client_key, ip, user_agent,
current_time(), "Refresh", True])
database.commit()
database.close()
def on_init(client_name, key):
Logger().write(LogVerbosity.Info, "Init slave: " + client_name)
if key != SecureSettings.get_string("master_key"):
Logger().write(LogVerbosity.Info, "Slave authentication failed")
disconnect()
return False
slave = APIController.slaves.get_slave(client_name)
if slave is not None:
if slave.connected:
Logger().write(
LogVerbosity.Info,
"Slave " + str(client_name) + " connected twice?")
disconnect()
return False
else:
Logger().write(LogVerbosity.Info,
"Slave " + str(client_name) + " reconnected")
slave.reconnect(request.sid)
APIController.slaves.changed()
return True
client = WebsocketClient(request.sid, current_time())
client.authenticated = True
APIController.slaves.add_slave(
SlaveClient(APIController.next_id(), client_name, client))
return client.authenticated
def check(self):
for pending in list(self.awaiting_messages):
if current_time() - pending.send_at > 10000:
Logger().write(LogVerbosity.All, "DHT message timeout")
self.node_timeout_handler(pending.message.ip,
pending.message.port)
pending.on_timeout()
self.awaiting_messages.remove(pending)
for received in list(self.received_messages):
if isinstance(received.message, QueryDHTMessage):
self.query_handler(received.ip, received.port,
received.message)
self.received_messages.remove(received)
continue
elif isinstance(received.message, ResponseDHTMessage):
pending = [
x for x in self.awaiting_messages
if x.message.message.transaction_id ==
received.message.transaction_id
]
if len(pending) == 0:
Logger().write(LogVerbosity.All,
"DHT response for no request (timed out?)")
self.received_messages.remove(received)
continue
Logger().write(LogVerbosity.All, "DHT message response")
pending[0].on_response(received.message) # answer to request
self.received_messages.remove(received)
self.awaiting_messages.remove(pending[0])
def update_peer(self, peer, from_state, to_state):
if from_state == PeerState.Starting and to_state == PeerState.Started:
self.connecting_peers.remove(peer)
self.connected_peers.append(peer)
elif from_state == PeerState.Starting and to_state == PeerState.Stopping:
self.connecting_peers.remove(peer)
self.cant_connect_peers.append(peer)
elif from_state == PeerState.Started and to_state == PeerState.Stopping:
self.connected_peers.remove(peer)
self.disconnected_peers.append(
(peer.uri, peer.source, current_time(), peer.counter.total))
self.high_speed_peers = len([
x for x in self.connected_peers if x.peer_speed == PeerSpeed.High
])
self.medium_speed_peers = len([
x for x in self.connected_peers if x.peer_speed == PeerSpeed.Medium
])
self.connected_peers = sorted(self.connected_peers,
key=lambda x: x.counter.value,
reverse=True)
self.disconnected_peers = sorted(self.disconnected_peers,
key=lambda x: x[3],
reverse=True)
self.connecting_peers_log = len(self.connecting_peers)
self.connected_peers_log = len(self.connected_peers)
self.cant_connect_peers_log = len(self.cant_connect_peers)
self.disconnected_peers_log = len(self.disconnected_peers)
def stop_slowest_peer(self):
if not self.should_stop_peers():
return True
if self.max_peers_connected - len(self.connected_peers) > 3:
return True # No need to stop slowest if we have enough room to connect more
peers_to_check = [
x for x in self.connected_peers
if current_time() - x.connection_manager.connected_on > 30000
]
peers_to_check = sorted(peers_to_check, key=lambda x: x.counter.value)
if len(peers_to_check) == 0:
return True
slowest = peers_to_check[0]
if slowest.counter.value > 100000: # if the slowest peer is downloading with more than 100kbps we're fine
return True
Logger().write(
LogVerbosity.Info,
str(slowest.id) +
" stopping peer to find a potential faster one. Peer speed last 5 seconds was "
+ str(write_size(slowest.counter.value)) + ", total: " +
str(write_size(slowest.counter.total)))
slowest.stop_async("Slowest")
return True
def update_node(self, id):
for bucket in self.buckets:
for node in bucket.nodes:
if node.byte_id == id:
node.seen()
bucket.last_update = current_time()
return
def stop(self):
run_time = current_time() - self.current_start
self.high = max(run_time, self.high)
self.low = min(run_time, self.low)
self.total_items += 1
self.total_time += run_time
self.average = self.total_time / self.total_items
def check_peer_state(self):
zero_speed_checks = 0
while self.state == PeerState.Started:
if current_time() - self.connection_manager.connected_on < 20000:
time.sleep(1)
continue
if not self.torrent.peer_manager.should_stop_peers():
time.sleep(1)
continue
if self.counter.total == 0:
Logger().write(LogVerbosity.Info,
str(self.id) + " stopping not downloading peer")
self.stop_async(
"Not downloading"
) # Stop since we haven't downloaded anything since connecting
return
if self.counter.value == 0:
zero_speed_checks += 1
if zero_speed_checks >= 10:
Logger().write(
LogVerbosity.Info,
str(self.id) +
" stopping currently not downloading peer")
self.stop_async(
"Not recently downloading"
) # Stop since we haven't downloaded anything in the last 10 seconds
return
else:
zero_speed_checks = 0
time.sleep(1)
def add_thread(self, thread):
thread.history_entry = ThreadEntry(thread.thread_name, current_time())
if thread.thread_name not in self.thread_history:
self.thread_history[thread.thread_name] = []
self.thread_history[thread.thread_name].append(thread.history_entry)
self.threads.append(thread)
self.thread_count = len(self.threads)
def announce_torrent(self, torrent):
self.last_announce = current_time()
announce_message = TrackerMessages.TrackerAnnounceMessage.for_http(torrent.info_hash, 2, torrent.total_size - torrent.left, torrent.left, torrent.uploaded, self.tracker_peer_request_amount)
path = self.uri.path + announce_message.as_param_string()
response = RequestFactory.make_request(path)
if response is None:
return False
try:
response_dict = Bencode.bdecode(response)
except BTFailure:
Logger().write(LogVerbosity.Info, 'Invalid tracker response: ' + str(response))
return False
if b"peers" not in response_dict:
return False
peers_data = response_dict[b"peers"]
total_peers = int(len(peers_data) / 6)
offset = 0
peers = []
for index in range(total_peers):
peers.append(uri_from_bytes(peers_data[offset:offset + 6]))
offset += 6
EventManager.throw_event(EventType.PeersFound, [peers, PeerSource.HttpTracker])
def execute(self):
while self.running:
Timing().start_timing("IO")
# Select in/outputs
input_peers = self.torrent.peer_manager.get_peers_for_reading()
received_messages = []
for peer in input_peers:
messages_size, messages = peer.connection_manager.handle_read()
if messages_size > 0:
self.average_download_counter.add_value(messages_size)
time = current_time()
received_messages += [(peer, x, time) for x in messages]
if len(received_messages) != 0:
self.torrent.message_processor.process_messages(received_messages)
for peer in self.torrent.peer_manager.connected_peers:
peer.download_manager.update_requests()
output_peers = self.torrent.peer_manager.get_peers_for_writing()
for peer in output_peers:
peer.connection_manager.handle_write()
Timing().stop_timing("IO")
sleep(0.005)
def runner(self):
while self.running:
sleep(0.01)
if not self.running:
break
self.last_tick = current_time()
self.tick()
def add_client(self, client_key, session_key, ip, user_agent):
if self.slave:
raise PermissionError("Cant call add_client on slave")
Logger().write(LogVerbosity.Debug, "Database add client")
database, cursor = self.connect()
time = current_time()
cursor.execute(
"INSERT INTO Clients (ClientKey, SessionKey, Issued, LastSeen) Values (?, ?, ?, ?)",
[client_key, session_key, time, time])
cursor.execute(
"INSERT INTO AuthorizeAttempts (ClientKey, IP, UserAgent, Timestamp, Type, Successful) VALUES (?, ?, ?, ?, ?, ?)",
[client_key, ip, user_agent,
current_time(), "Login", True])
database.commit()
database.close()
def _start_playing_torrent(self):
if Settings.get_bool("slave"):
self.history_id, = SlaveClientController.request_master(
"add_watched_torrent", 5, self.media_data.type,
self.media_data.title, self.media_data.id, self.torrent.uri,
self.torrent.media_file.path, self.media_data.image,
self.media_data.season, self.media_data.episode,
current_time())
else:
self.history_id = Database().add_watched_torrent(
self.media_data.type, self.media_data.title,
self.media_data.id, self.torrent.uri,
self.torrent.media_file.path, self.media_data.image,
self.media_data.season, self.media_data.episode,
current_time())
VLCPlayer().play("http://localhost:50009/torrent",
self.media_data.start_from)
def add_node(self, node):
if len(self.nodes) >= self.max_nodes:
self.nodes = [
x for x in self.nodes if x.node_state != NodeState.Bad
]
self.nodes.append(node)
self.last_changed = current_time()
