def __init__(self, tracker_address, video_name, user_name): """ Create a new P2PUser. Set the packet size, instantiate the manager, and establish clients. Currently, the clients are static but will become dynamic when the tracker is implemented. """ self.packet_size = 1000 self.user_name = user_name self.my_ip = user_name self.my_port = 0 register_to_tracker_as_user(tracker_address, self.my_ip, self.my_port, video_name) # Connect to the server # Cache will get a response when each chunk is downloaded from the server. # Note that this flag should **NOT** be set for the caches, as the caches # downloads will be aborted after 8 seconds with no expectation. # After the cache download period, the files themselves will be checked # to see what remains to be downloaded from the server. server_ip_address = retrieve_server_address_from_tracker(tracker_address) self.server_client = ThreadClient(self, server_ip_address, self.packet_size) self.server_client.set_respond_RETR(True) self.tracker_address = tracker_address self.clients = [] self.num_of_caches = num_of_caches self.manager = None # TODO: create the manager class to decode/play self.info_thread = '' # Thread for exchanging information
def update_info(self): user = self.user video_name = self.video_name filename = self.filename code_param_n = self.code_param_n code_param_k = self.code_param_k filename = 'file-' + self.video_name + '.' + str(1) folder_name = 'video-' + self.video_name + '/' + self.video_name + '.' + str(1) + '.dir/' inst_CNKS = 'CNKS ' + self.filename inst_UPDG = 'UPDG ' # self.clients : list of connected guys T_buffer = CACHE_DOWNLOAD_DURATION + SERVER_DOWNLOAD_DURATION ct_period = int(T_buffer / T_update_info) ct_loop = ct_period - 1 clients_copy = [] while True: if not self.flag: print '[infoThread.py] Thread terminates' break ct_loop += 1 if ct_loop == ct_period: # Copy self.clients to client_copy while clients_copy: each_client = clients_copy.pop() each_client.put_instruction('QUIT') for each in self.user.clients: each_ip = each.address each_client = ThreadClient(self, each_ip, self.user.packet_size, 1) clients_copy.append(each_client) each_client.put_instruction('ID %s' % self.user.user_name) ct_loop = 0 available_chunks = [0]*len(clients_copy) # available_chunks[i] = cache i's availble chunks rates = [0]*len(clients_copy) # rates[i] = cache i's offered rate union_chunks = [] # union of all available indices for i in range(len(clients_copy)): client = clients_copy[i] client.put_instruction(inst_CNKS) return_str = client.get_response().split('&') if return_str[0] == '': available_chunks[i] = [] else: available_chunks[i] = map(str, return_str[0].split('%')) for j in range(len(available_chunks[i])): available_chunks[i][j] = available_chunks[i][j].zfill(2) rates[i] = int(return_str[1]) union_chunks = list( set(union_chunks) | set(available_chunks[i]) ) ## index assignment here # Assign chunks to cache using cache_chunks_to_request. if DEBUGGING_MSG: print '[user.py] Update_info_loop : Rates ', rates print '[user.py] Update_info_loop : Available chunks', available_chunks assigned_chunks = cache_chunks_to_request(available_chunks, rates, self.code_param_n, self.code_param_k) effective_rates = [0]*len(rates) for i in range(len(rates)): effective_rates[i] = len(assigned_chunks[i]) chosen_chunks = [j for i in assigned_chunks for j in i] flag_deficit = int(sum(effective_rates) < code_param_k) # True if user needs more rate from caches # request assigned chunks for i in range(len(clients_copy)): client = clients_copy[i] print "[user.py] Update_info_loop : [Client " + str(i) + "] flag_deficit: ", flag_deficit, \ ", Assigned chunks: ", assigned_chunks[i] client.put_instruction(inst_UPDG + str(flag_deficit)) sleep(T_update_info) # Close all connections while clients_copy: each_client = clients_copy.pop() each_client.put_instruction('QUIT')
def download(self, video_name, start_frame): print '[user.py] P2Puser starts downloading' connected_caches = [] self.not_connected_caches = not_connected_caches = [] # Connect to the caches cache_ip_addr = retrieve_caches_address_from_tracker(self.tracker_address, 100, self.user_name) #cache_ip_addr[0][0] = '[' + cache_ip_addr[0][0] + ']' self.cache_ip_addr = cache_ip_addr self.num_of_caches = min(self.num_of_caches, len(cache_ip_addr)) choke_state = 0 # 0 : usual state, 1 : overhead state choke_ct = 0 for i in range(self.num_of_caches): each_client = ThreadClient(self, cache_ip_addr[i], self.packet_size, i) each_client.put_instruction('ID %s' % self.user_name) self.clients.append(each_client) connected_caches.append(each_client) print '[user.py] ', i, 'th connection is CONNECTED : ' , cache_ip_addr[i] if DEBUG_RYAN: pdb.set_trace() for i in range(self.num_of_caches, len(cache_ip_addr)): #Is it not entering this statement here? if DEBUG_RYAN: pdb.set_trace() each_client = ThreadClient(self, cache_ip_addr[i], self.packet_size, i) each_client.put_instruction('ID %s' % self.user_name) not_connected_caches.append(each_client) print '[user.py] ', i, 'th connection is RESERVED: ' , cache_ip_addr[i] available_chunks = set([]) print '[user.py] putting VLEN', video_name self.clients[0].put_instruction('VLEN file-%s' % (video_name)) print '[user.py] retrieving VLEN' vlen_str = self.clients[0].get_response().split('\n')[0] vlen_items = vlen_str.split('&') print "VLEN: ", vlen_items num_frames, code_param_n, code_param_k = int(vlen_items[0]), int(vlen_items[4]), int(vlen_items[5]) base_file_name = video_name + '.mkv' #turning it into an .mvk also works. Probably should store what kind of file it is server side #or just make everything .mkv. .MKV is a container file for video, audio, and other stuff. #Read here for a nice description: #http://lifehacker.com/5893250/whats-the-difference-between-all-these-video-formats-and-which-one-should-i-use #base_file_name = video_name + '.mkv' try: os.mkdir('video-' + video_name) except: pass # Set internal chunk_size through putting an internal instruction into # the queue. base_file = open('video-' + video_name + '/' + base_file_name, 'ab') base_file_full_path = os.path.abspath('video-' + video_name + '/' + base_file_name) self.info_thread = infoThread(video_name, code_param_n, code_param_k, self) self.info_thread.flag = True self.info_thread.start() for frame_number in xrange(start_frame, num_frames + 1): sys.stdout.flush() effective_rates = [0]*len(self.clients) assigned_chunks = [0]*len(self.clients) if frame_number < num_frames: # Usual frames inst_INTL = 'INTL ' + 'CNKN ' + vlen_items[2] # chunk size of typical frame (not last one) for client in self.clients: client.put_instruction(inst_INTL) self.server_client.put_instruction(inst_INTL) else: # Last frame inst_INTL = 'INTL ' + 'CNKN ' + vlen_items[3] # chunk size of last frame for client in self.clients: client.put_instruction(inst_INTL) self.server_client.put_instruction(inst_INTL) print '[user.py] frame_number : ', frame_number filename = 'file-' + video_name + '.' + str(frame_number) # directory for this frame folder_name = 'video-' + video_name + '/' + video_name + '.' + str(frame_number) + '.dir/' # get available chunks lists from cache A and B. inst_CNKS = 'CNKS ' + filename inst_RETR = 'RETR ' + filename inst_UPDG = 'UPDG ' inst_NOOP = 'NOOP' inst_CACHEDATA = 'CACHEDATA ' ###### DECIDING WHICH CHUNKS TO DOWNLOAD FROM CACHES: TIME 0 ###### if DEBUG_RYAN: pdb.set_trace() available_chunks = [0]*len(self.clients) # available_chunks[i] = cache i's availble chunks rates = [0]*len(self.clients) # rates[i] = cache i's offered rate union_chunks = [] # union of all available indices for i in range(len(self.clients)): client = self.clients[i] client.put_instruction(inst_CNKS) return_str = client.get_response().split('&') if return_str[0] == '': available_chunks[i] = [] else: available_chunks[i] = map(str, return_str[0].split('%')) for j in range(len(available_chunks[i])): available_chunks[i][j] = available_chunks[i][j].zfill(2) rates[i] = int(return_str[1]) union_chunks = list( set(union_chunks) | set(available_chunks[i]) ) ## index assignment here # Assign chunks to cache using cache_chunks_to_request. print '[user.py] Rates ', rates print '[user.py] Available chunks', available_chunks assigned_chunks = cache_chunks_to_request(available_chunks, rates, code_param_n, code_param_k) effective_rates = [0]*len(rates) for i in range(len(rates)): effective_rates[i] = len(assigned_chunks[i]) chosen_chunks = [j for i in assigned_chunks for j in i] flag_deficit = int(sum(effective_rates) < code_param_k) # True if user needs more rate from caches list_of_cache_requests = [] # request assigned chunks for i in range(len(self.clients)): client = self.clients[i] client_ip_address = client.address[0] + ':' + str(client.address[1]) print '[user.py] Server_request 2 = "' , assigned_chunks[i] , '"' client_request_string = '%'.join(assigned_chunks[i]) + '&1' print "[user.py] [Client " + str(i) + "] flag_deficit: ", flag_deficit, \ ", Assigned chunks: ", assigned_chunks[i], \ ", Request string: ", client_request_string if DEBUG_RYAN: pdb.set_trace() cachedata_request_string = client_request_string.replace('&1','&' + client_ip_address) list_of_cache_requests.append(filename+ '.' + cachedata_request_string) client.put_instruction(inst_UPDG + str(flag_deficit)) client.put_instruction(inst_RETR + '.' + client_request_string) if False: # WHY IS THIS NOT WORKING? if not assigned_chunks[i]: pass #client.put_instruction(inst_NOOP) else: client.put_instruction(inst_RETR + '.' + client_request_string) ###### DECIDING CHUNKS THAT HAVE TO BE DOWNLOADED FROM CACHE: TIME 0 ###### # Before CACHE_DOWNLOAD_DURATION, also start requesting chunks from server. server_request = [] server_request_2 = [] cdrs = '_' .join(list_of_cache_requests) #cache data request string. Used for parsing inside of server.py's ftp_CACHEDATA if frame_number < num_frames: size_of_chunks = vlen_items[2] else: size_of_chunks = vlen_items[3] cdrs = cdrs + '?' + str(size_of_chunks) chosen_chunks = list(chosen_chunks) num_chunks_rx_predicted = len(chosen_chunks) server_request = chunks_to_request(chosen_chunks, range(0, code_param_n), code_param_k - num_chunks_rx_predicted) num_of_chks_from_server = len(server_request) if num_of_chks_from_server == 0: self.server_client.put_instruction(inst_NOOP) print '[user.py] Caches handling code_param_k chunks, so no request to server. Sending a NOOP' else: print '[user.py] Server_request = "' , server_request , '"' server_request_string = '%'.join(server_request) + '&1' if DEBUG_RYAN: pdb.set_trace() self.server_client.put_instruction(inst_CACHEDATA + cdrs) self.server_client.put_instruction(inst_RETR + '.' + server_request_string) if(DEBUGGING_MSG): print "[user.py] Requesting from server: ", server_request, ", Request string: ", server_request_string print "[user.py] Sending to server's ftp_CACHEDATA: ", inst_CACHEDATA + cdrs #update_server_load(tracker_address, video_name, num_of_chks_from_server) sleep(CACHE_DOWNLOAD_DURATION) ###### STOPPING CACHE DOWNLOADS: TIME 8 (CACHE_DOWNLOAD_DURATION) ###### # immediately stop cache downloads. for client in self.clients: try: client.client.abort() except: print "[user.py] Cache connections suddenly aborted. Stopping all download." return print "[user.py] Cache connections aborted for frame %d" % (frame_number) ###### REQUEST ADDITIONAL CHUNKS FROM SERVER: TIME 8 (CACHE_DOWNLOAD_DURATION) ###### # Request from server remaining chunks missing # Look up the download directory and count the downloaded chunks chunk_nums_rx = chunk_nums_in_frame_dir(folder_name) if (DEBUGGING_MSG): print "%d chunks received so far for frame %d: " % (len(chunk_nums_rx), frame_number) print chunk_nums_rx # Add the chunks that have already been requested from server chunk_nums_rx = list (set(chunk_nums_in_frame_dir(folder_name)) | set(server_request)) print "[user.py] chunk_nums_rx", chunk_nums_rx addtl_server_request = [] num_chunks_rx = len(chunk_nums_rx) if (num_chunks_rx >= code_param_k): print "[user.py] No additional chunks to download from the server. Sending a NOOP" self.server_client.put_instruction(inst_NOOP) else: addtl_server_request = chunks_to_request(chunk_nums_rx, range(0, code_param_n), code_param_k - num_chunks_rx) print "[user.py] addtl_server_requests", addtl_server_request if addtl_server_request: addtl_server_request_string = '%'.join(addtl_server_request) + '&1' # The last digit '1' means 'I am user' # server should always be set with flag_deficit = 0 (has all chunks) self.server_client.put_instruction(inst_RETR + '.' + addtl_server_request_string) if(DEBUGGING_MSG): print "[user.py] Requesting from server: ", addtl_server_request elif (DEBUGGING_MSG): print "No unique chunks from server requested." ###### WAIT FOR CHUNKS FROM SERVER TO FINISH DOWNLOADING: TIME 10 ###### sleep(SERVER_DOWNLOAD_DURATION) if (DEBUGGING_MSG): print "[user.py] Waiting to receive all elements from server." if frame_number > start_frame and (server_request or addtl_server_request) and VLC_PLAYER_USE: # Need to pause it! self.VLC_pause_video() if server_request: resp_RETR = self.server_client.get_response() parsed_form = parse_chunks(resp_RETR) fname, framenum, chunks, user_or_cache = parsed_form print "[user.py] Downloaded chunks from server: ", chunks if addtl_server_request: resp_RETR = self.server_client.get_response() parsed_form = parse_chunks(resp_RETR) fname, framenum, chunks, user_or_cache = parsed_form print "[user.py] Downloaded chunks from server: ", chunks # Now play it if frame_number > start_frame and (server_request or addtl_server_request) and VLC_PLAYER_USE: self.VLC_pause_video() chunk_nums = chunk_nums_in_frame_dir(folder_name) num_chunks_rx = len(chunk_nums) if num_chunks_rx >= code_param_k and DEBUGGING_MSG: print "[user.py] Received", code_param_k, "packets" else: print "[user.py] Did not receive", code_param_k, "packets for this frame." # abort the connection to the server self.server_client.client.abort() # put together chunks into single frame; then concatenate onto original file. print 'about to decode...' chunksList = chunk_files_in_frame_dir(folder_name) if frame_number != start_frame: print 'size of base file:', os.path.getsize('video-' + video_name + '/' + base_file_name) print 'trying to decode' filefec.decode_from_files(base_file, chunksList) print 'decoded. Size of base file =', os.path.getsize('video-' + video_name + '/' + base_file_name) if frame_number == 1 and VLC_PLAYER_USE: self.VLC_empty_list() self.VLC_start_video(base_file_full_path) if USER_TOPOLOGY_UPDATE: if choke_state == 0: # Normal state print '[user.py] Normal state : ', choke_ct choke_ct += 1 if choke_ct == T_choke: choke_ct = 0 if len(not_connected_caches) == 0: pass else: # Add a new cache temporarily new_cache_index = random.sample(range(len(not_connected_caches)), 1) if new_cache_index >= 0: new_cache = not_connected_caches[new_cache_index[0]] self.clients.append(new_cache) connected_caches.append(new_cache) not_connected_caches.remove(new_cache) print '[user.py] Topology Update : Temporarily added ', new_cache.address choke_state = 1 # Now, move to transitional state choke_ct = 0 print '[user.py] Topology Update : Now the state is changed to overhead staet' #print '[user.py]', connected_caches, not_connected_caches, self.clients print '[user.py] conneced caches', self.clients elif choke_state == 1: # Overhead state print '[user.py] Overhead state : ', choke_ct choke_ct += 1 if choke_ct == T_choke2: # Temporary period to spend with temporarily added node rate_vector = [0] * len(self.clients) p_vector = [0] * len(self.clients) for i in range(len(self.clients)): rate_vector[i] = len(assigned_chunks[i]) p_vector[i] = math.exp( -eps_choke * rate_vector[i]) p_sum = sum(p_vector) for i in range(len(self.clients)): p_vector[i] /= p_sum cdf = [(0,0)] * len(self.clients) cdf[0] = (0, 0) for i in range(1, len(self.clients)): cdf[i] = (i, cdf[i-1][1] + p_vector[i-1]) print '[user.py] cdf :', cdf client_index = max(i for r in [random.random()] for i,c in cdf if c <= r) # http://stackoverflow.com/questions/4265988/generate-random-numbers-with-a-given-numerical-distribution removed_cache = self.clients[client_index] #removed_cache.put_instruction('QUIT') self.clients.remove(removed_cache) connected_caches.remove(removed_cache) not_connected_caches.append(removed_cache) print '[user.py] Topology Update : ', removed_cache.address, 'is chocked.' choke_state = 0 # Now, move to normal state choke_ct = 0
class P2PUser(): #def __init__(self, tracker_address, video_name, packet_size): def __init__(self, tracker_address, video_name, user_name): """ Create a new P2PUser. Set the packet size, instantiate the manager, and establish clients. Currently, the clients are static but will become dynamic when the tracker is implemented. """ self.packet_size = 1000 self.user_name = user_name self.my_ip = user_name self.my_port = 0 register_to_tracker_as_user(tracker_address, self.my_ip, self.my_port, video_name) # Connect to the server # Cache will get a response when each chunk is downloaded from the server. # Note that this flag should **NOT** be set for the caches, as the caches # downloads will be aborted after 8 seconds with no expectation. # After the cache download period, the files themselves will be checked # to see what remains to be downloaded from the server. server_ip_address = retrieve_server_address_from_tracker(tracker_address) self.server_client = ThreadClient(server_ip_address, self.packet_size) self.server_client.set_respond_RETR(True) self.tracker_address = tracker_address self.clients = [] self.num_of_caches = num_of_caches self.manager = None # TODO: create the manager class to decode/play def VLC_start_video(self, video_path): # Put the file into the queue and play it url = 'http://127.0.0.1:8080/requests/status.xml?command=in_play&input=file://' url = url + video_path print '[user.py] ', url urllib2.urlopen(url).read() def VLC_pause_video(self): # Pause or play it url = 'http://127.0.0.1:8080/requests/status.xml?command=pl_pause' print '[user.py] ', url urllib2.urlopen(url).read() def play(self, video_name, frame_number): """ Starts playing the video as identified by either name or number and begins handling the data connections necessary to play the video, starting at frame_number (the 10-second section of time within the vid). """ # inform the web browser we have started playing if not self.manager.playing(): self.manager.start_playing() # TODO: add decoding. def connected_caches(): f = open(config_file) fs = csv.reader(f, delimiter = ' ') for row in fs: if (DEBUGGING_MSG): print '[server.py] Loading movie : ', row movie_name = row[0] self.movies_LUT[movie_name] = (int(row[1]), int(row[2]), int(row[3]), int(row[4]), int(row[5]), int(row[6])) def download(self, video_name, start_frame): connected_caches = [] not_connected_caches = [] # Connect to the caches cache_ip_addr = retrieve_caches_address_from_tracker(self.tracker_address, 100, self.user_name) self.cache_ip_addr = cache_ip_addr #connected_caches = set([]) self.num_of_caches = min(self.num_of_caches, len(cache_ip_addr)) #connected_caches_index = [0] * self.num_of_caches #not_connected_caches = set(range(len(cache_ip_addr))) choke_state = 0 # 0 : usual state, 1 : overhead state choke_ct = 0 for i in range(self.num_of_caches): each_client = ThreadClient(cache_ip_addr[i], self.packet_size, i) self.clients.append(each_client) connected_caches.append(each_client) print '[user.py] ', i, 'th connection is CONNECTED : ' , cache_ip_addr[i] for i in range(self.num_of_caches, len(cache_ip_addr)): each_client = (cache_ip_addr[i], self.packet_size, i) not_connected_caches.append(each_client) print '[user.py] ', i, 'th connection is RESERVED: ' , cache_ip_addr[i] available_chunks = set([]) self.clients[0].put_instruction('VLEN file-%s' % (video_name)) vlen_str = self.clients[0].get_response().split('\n')[0] vlen_items = vlen_str.split('&') print "VLEN: ", vlen_items num_frames = int(vlen_items[0]) base_file_name = video_name + '.flv' try: os.mkdir('video-' + video_name) except: pass # Set internal chunk_size through putting an internal instruction into # the queue. base_file = open('video-' + video_name + '/' + base_file_name, 'ab') base_file_full_path = os.path.abspath('video-' + video_name + '/' + base_file_name) frame_number = 1 for i in range(30): sys.stdout.flush() effective_rates = [0]*len(self.clients) assigned_chunks = [0]*len(self.clients) if frame_number < num_frames: # Usual frames inst_INTL = 'INTL ' + 'CNKN ' + vlen_items[2] # chunk size of typical frame (not last one) for client in self.clients: client.put_instruction(inst_INTL) self.server_client.put_instruction(inst_INTL) else: # Last frame inst_INTL = 'INTL ' + 'CNKN ' + vlen_items[3] # chunk size of last frame for client in self.clients: client.put_instruction(inst_INTL) self.server_client.put_instruction(inst_INTL) print '[user.py] frame_number : ', frame_number filename = 'file-' + video_name + '.' + str(frame_number) # directory for this frame folder_name = 'video-' + video_name + '/' + video_name + '.' + str(frame_number) + '.dir/' # FAKE USER is deleting all chunks and receiving again again chunk_delete_all_in_frame_dir(folder_name) print '[user.py] deleting successfully...' # get available chunks lists from cache A and B. inst_CNKS = 'CNKS ' + filename inst_RETR = 'RETR ' + filename inst_NOOP = 'NOOP' ###### DECIDING WHICH CHUNKS TO DOWNLOAD FROM CACHES: TIME 0 ###### available_chunks = [0]*len(self.clients) # available_chunks[i] = cache i's availble chunks rates = [0]*len(self.clients) # rates[i] = cache i's offered rate union_chunks = [] # union of all available indices for i in range(len(self.clients)): client = self.clients[i] client.put_instruction(inst_CNKS) return_str = client.get_response().split('&') if return_str[0] == '': available_chunks[i] = [] else: available_chunks[i] = map(str, return_str[0].split('%')) for j in range(len(available_chunks[i])): available_chunks[i][j] = available_chunks[i][j].zfill(2) rates[i] = int(return_str[1]) union_chunks = list( set(union_chunks) | set(available_chunks[i]) ) ## index assignment here # Assign chunks to cache using cache_chunks_to_request. print '[user.py] Rates ', rates print '[user.py] Available chunks', available_chunks assigned_chunks = cache_chunks_to_request(available_chunks, rates) effective_rates = [0]*len(rates) for i in range(len(rates)): effective_rates[i] = len(assigned_chunks[i]) chosen_chunks = [j for i in assigned_chunks for j in i] flag_deficit = (sum(effective_rates) < 20) # True if user needs more rate from caches # request assigned chunks for i in range(len(self.clients)): client = self.clients[i] client_request_string = '%'.join(assigned_chunks[i]) client_request_string = client_request_string + '&' + str(int(flag_deficit)) print "[user.py] [Client " + str(i) + "] flag_deficit: ", int(flag_deficit), \ ", Assigned chunks: ", assigned_chunks[i], \ ", Request string: ", client_request_string client.put_instruction(inst_RETR + '.' + client_request_string) ###### DECIDING CHUNKS THAT HAVE TO BE DOWNLOADED FROM CACHE: TIME 0 ###### # Before CACHE_DOWNLOAD_DURATION, also start requesting chunks from server. server_request = [] chosen_chunks = list(chosen_chunks) num_chunks_rx_predicted = len(chosen_chunks) server_request = chunks_to_request(chosen_chunks, range(0, 40), 20 - num_chunks_rx_predicted) num_of_chks_from_server = len(server_request) if num_of_chks_from_server == 0: self.server_client.put_instruction(inst_NOOP) print '[user.py] Caches handling 20 chunks, so no request to server. Sending a NOOP' else: server_request_string = '%'.join(server_request) server_request_string = server_request_string + '&' + str(1) ## DOWNLOAD FROM SERVER : binary_g = 1 self.server_client.put_instruction(inst_RETR + '.' + server_request_string) if(DEBUGGING_MSG): print "[user.py] Requesting from server: ", server_request, ", Request string: ", server_request_string #update_server_load(tracker_address, video_name, num_of_chks_from_server) sleep(CACHE_DOWNLOAD_DURATION) ###### STOPPING CACHE DOWNLOADS: TIME 8 (CACHE_DOWNLOAD_DURATION) ###### # immediately stop cache downloads. for client in self.clients: try: client.client.abort() except: print "[user.py] Cache connections suddenly aborted. Stopping all download." return print "[user.py] Cache connections aborted for frame %d" % (frame_number) ###### REQUEST ADDITIONAL CHUNKS FROM SERVER: TIME 8 (CACHE_DOWNLOAD_DURATION) ###### # Request from server remaining chunks missing # Look up the download directory and count the downloaded chunks chunk_nums_rx = chunk_nums_in_frame_dir(folder_name) if (DEBUGGING_MSG): print "%d chunks received so far for frame %d: " % (len(chunk_nums_rx), frame_number) print chunk_nums_rx # Add the chunks that have already been requested from server chunk_nums_rx = list (set(chunk_nums_in_frame_dir(folder_name)) | set(server_request)) addtl_server_request = [] num_chunks_rx = len(chunk_nums_rx) if (num_chunks_rx >= 20): print "[user.py] No additional chunks to download from the server. Sending a NOOP" self.server_client.put_instruction(inst_NOOP) else: addtl_server_request = chunks_to_request(chunk_nums_rx, range(0, 40), 20 - num_chunks_rx) if addtl_server_request: addtl_server_request_string = '%'.join(addtl_server_request) # server should always be set with flag_deficit = 0 (has all chunks) addtl_server_request_string = addtl_server_request_string + '&' + str(1) ## DOWNLOAD FROM SERVER : binary_g = 1 self.server_client.put_instruction(inst_RETR + '.' + addtl_server_request_string) if(DEBUGGING_MSG): print "[user.py] Requesting from server: ", addtl_server_request elif (DEBUGGING_MSG): print "No unique chunks from server requested." ###### WAIT FOR CHUNKS FROM SERVER TO FINISH DOWNLOADING: TIME 10 ###### sleep(SERVER_DOWNLOAD_DURATION) if (DEBUGGING_MSG): print "[user.py] Waiting to receive all elements from server." if frame_number > start_frame and (server_request or addtl_server_request) and VLC_PLAYER_USE: # Need to pause it! self.VLC_pause_video() if server_request: resp_RETR = self.server_client.get_response() parsed_form = parse_chunks(resp_RETR) fname, framenum, binary_g, chunks = parsed_form print "[user.py] Downloaded chunks from server: ", chunks if addtl_server_request: resp_RETR = self.server_client.get_response() parsed_form = parse_chunks(resp_RETR) fname, framenum, binary_g, chunks = parsed_form print "[user.py] Downloaded chunks from server: ", chunks # Now play it if frame_number > start_frame and (server_request or addtl_server_request) and VLC_PLAYER_USE: self.VLC_pause_video() chunk_nums = chunk_nums_in_frame_dir(folder_name) num_chunks_rx = len(chunk_nums) if num_chunks_rx >= 20 and DEBUGGING_MSG: print "[user.py] Received 20 packets" else: print "[user.py] Did not receive 20 packets for this frame." # abort the connection to the server self.server_client.client.abort() # put together chunks into single frame; then concatenate onto original file. print 'about to decode...' chunksList = chunk_files_in_frame_dir(folder_name) if frame_number != start_frame: print 'size of base file:', os.path.getsize('video-' + video_name + '/' + base_file_name) print 'trying to decode' #filefec.decode_from_files(base_file, chunksList) print 'decoded. Size of base file =', os.path.getsize('video-' + video_name + '/' + base_file_name) if frame_number == 1 and VLC_PLAYER_USE: self.VLC_start_video(base_file_full_path) if USER_TOPOLOGY_UPDATE: if choke_state == 0: # Normal state print '[user.py] Normal state : ', choke_ct choke_ct += 1 if choke_ct == T_choke: choke_ct = 0 if len(not_connected_caches) == 0: pass else: # Add a new cache temporarily new_cache_index = random.sample(range(len(not_connected_caches)), 1) if new_cache_index >= 0: new_cache_meta = not_connected_caches[new_cache_index[0]] new_cache = ThreadClient(*new_cache_meta) self.clients.append(new_cache) connected_caches.append(new_cache) not_connected_caches.remove(new_cache_meta) print '[user.py] Topology Update : Temporarily added ', new_cache.address choke_state = 1 # Now, move to transitional state choke_ct = 0 print '[user.py] Topology Update : Now the state is changed to overhead staet' #print '[user.py]', connected_caches, not_connected_caches, self.clients print '[user.py] conneced caches', self.clients elif choke_state == 1: # Overhead state print '[user.py] Overhead state : ', choke_ct choke_ct += 1 if choke_ct == T_choke2: # Temporary period to spend with temporarily added node rate_vector = [0] * len(self.clients) p_vector = [0] * len(self.clients) for i in range(len(self.clients)): rate_vector[i] = len(assigned_chunks[i]) p_vector[i] = math.exp( -eps_choke * rate_vector[i]) # >>> cdf = [(1, 0), (2,0.1), (3,0.15), (4,0.2), (5,0.4), (6,0.8)] p_sum = sum(p_vector) for i in range(len(self.clients)): p_vector[i] /= p_sum cdf = [(0,0)] * len(self.clients) cdf[0] = (0, 0) for i in range(1, len(self.clients)): cdf[i] = (i, cdf[i-1][1] + p_vector[i-1]) print '[user.py] cdf :', cdf client_index = max(i for r in [random.random()] for i,c in cdf if c <= r) # http://stackoverflow.com/questions/4265988/generate-random-numbers-with-a-given-numerical-distribution # client_index = rate_vector.index(min(rate_vector)) removed_cache = self.clients[client_index] removed_cache.put_instruction('QUIT') self.clients.remove(removed_cache) connected_caches.remove(removed_cache) new_cache_meta = (self.cache_ip_addr[client_index], 1000, client_index) not_connected_caches.append(new_cache_meta) print '[user.py] Topology Update : ', removed_cache.address, 'is chocked.' choke_state = 0 # Now, move to normal state choke_ct = 0 def disconnect(self, tracker_address, video_name, user_name): for client in self.clients: client.put_instruction('QUIT') self.server_client.put_instruction('QUIT') print "[user.py] Closed all connections." my_ip = user_name my_port = 0 my_video_name = video_name deregister_to_tracker_as_user(tracker_address, my_ip, my_port, video_name) print "[user.py] BYE" sys.stdout.flush()
class P2PUser(): def __init__(self, tracker_address, video_name, user_name, session = None): """ Create a new P2PUser. Set the packet size, instantiate the manager, and establish clients. Currently, the clients are static but will become dynamic when the tracker is implemented. """ self.packet_size = 1000 self.user_name = user_name self.my_ip = user_name self.my_port = 0 register_to_tracker_as_user(tracker_address, self.my_ip, self.my_port, video_name,session) # Connect to the server # Cache will get a response when each chunk is downloaded from the server. # Note that this flag should **NOT** be set for the caches, as the caches # downloads will be aborted after 8 seconds with no expectation. # After the cache download period, the files themselves will be checked # to see what remains to be downloaded from the server. server_ip_address = retrieve_server_address_from_tracker(tracker_address, session) self.server_client = ThreadClient(self, server_ip_address, self.packet_size) self.server_client.set_respond_RETR(True) self.tracker_address = tracker_address self.clients = [] self.num_of_caches = num_of_caches self.manager = None # TODO: create the manager class to decode/play self.info_thread = '' # Thread for exchanging information def VLC_start_video(self, video_path): # Put the file into the queue and play it url = 'http://127.0.0.1:8081/requests/status.xml?command=in_play&input=file://' url = url + video_path urllib2.urlopen(url).read() def VLC_pause_video(self): # Pause or play it url = 'http://127.0.0.1:8081/requests/status.xml?command=pl_pause' urllib2.urlopen(url).read() def VLC_empty_list(self): # Empty playlist url = 'http://127.0.0.1:8081/requests/status.xml?command=pl_empty' urllib2.urlopen(url).read() def play(self, video_name, frame_number): """ Starts playing the video as identified by either name or number and begins handling the data connections necessary to play the video, starting at frame_number (the 10-second section of time within the vid). """ # inform the web browser we have started playing if not self.manager.playing(): self.manager.start_playing() # TODO: add decoding. def connected_caches(): f = open(config_file) fs = csv.reader(f, delimiter = ' ') for row in fs: if (DEBUGGING_MSG): print '[server.py] Loading movie : ', row movie_name = row[0] self.movies_LUT[movie_name] = (int(row[1]), int(row[2]), int(row[3]), int(row[4]), int(row[5]), int(row[6])) def download(self, video_name, start_frame, session=None): global global_frame_number print '[user.py] P2Puser starts downloading' connected_caches = [] self.not_connected_caches = not_connected_caches = [] inst_SENDPORT = 'SENDPORT ' if DEBUG_FULL: pdb.set_trace() #str(self.my_port) self.my_ip video_name user self.server_client.put_instruction(inst_SENDPORT + str(self.my_port) + ' ' + self.my_ip + ' ' + video_name + ' user') # Connect to the caches cache_ip_addr = retrieve_caches_address_from_tracker(self.tracker_address, 100, self.user_name, session = session) self.cache_ip_addr = cache_ip_addr self.num_of_caches = min(self.num_of_caches, len(cache_ip_addr)) choke_state = 0 # 0 : usual state, 1 : overhead state choke_ct = 0 for i in range(self.num_of_caches): each_client = ThreadClient(self, cache_ip_addr[i], self.packet_size, i) each_client.put_instruction('ID %s' % self.user_name) self.clients.append(each_client) connected_caches.append(each_client) print '[user.py] ', i, 'th connection is CONNECTED : ' , cache_ip_addr[i] if DEBUG_FULL: pdb.set_trace() for i in range(self.num_of_caches, len(cache_ip_addr)): #Is it not entering this statement here? if DEBUG_FULL: pdb.set_trace() each_client = ThreadClient(self, cache_ip_addr[i], self.packet_size, i) each_client.put_instruction('ID %s' % self.user_name) not_connected_caches.append(each_client) print '[user.py] ', i, 'th connection is RESERVED: ' , cache_ip_addr[i] available_chunks = set([]) print '[user.py] putting VLEN', video_name self.clients[0].put_instruction('VLEN file-%s' % (video_name)) print '[user.py] retrieving VLEN' vlen_str = self.clients[0].get_response().split('\n')[0] vlen_items = vlen_str.split('&') print "VLEN: ", vlen_items num_frames, code_param_n, code_param_k = int(vlen_items[0]), int(vlen_items[4]), int(vlen_items[5]) base_file_name = video_name + '.mkv' #turning it into an .mvk also works. Probably should store what kind of file it is server side #or just make everything .mkv. .MKV is a container file for video, audio, and other stuff. #Read here for a nice description: #http://lifehacker.com/5893250/whats-the-difference-between-all-these-video-formats-and-which-one-should-i-use #base_file_name = video_name + '.mkv' try: os.mkdir('video-' + video_name) except: pass # Set internal chunk_size through putting an internal instruction into # the queue. base_file = open('video-' + video_name + '/' + base_file_name, 'ab') base_file_full_path = os.path.abspath('video-' + video_name + '/' + base_file_name) self.info_thread = infoThread(video_name, code_param_n, code_param_k, self) self.info_thread.flag = True self.info_thread.start() for frame_number in range(start_frame, num_frames + 1): global_frame_number = frame_number sys.stdout.flush() effective_rates = [0]*len(self.clients) assigned_chunks = [0]*len(self.clients) if frame_number < num_frames: # Usual frames inst_INTL = 'INTL ' + 'CNKN ' + vlen_items[2] # chunk size of typical frame (not last one) for client in self.clients: client.put_instruction(inst_INTL) self.server_client.put_instruction(inst_INTL) else: # Last frame inst_INTL = 'INTL ' + 'CNKN ' + vlen_items[3] # chunk size of last frame for client in self.clients: client.put_instruction(inst_INTL) self.server_client.put_instruction(inst_INTL) print '[cache.py -debug] Requesting frame ==================' print '[user.py -debug] current_time =', strftime("%Y-%m-%d %H:%M:%S") print '[user.py] frame_number requesting: ', frame_number filename = 'file-' + video_name + '.' + str(frame_number) # directory for this frame folder_name = 'video-' + video_name + '/' + video_name + '.' + str(frame_number) + '.dir/' # get available chunks lists from cache A and B. inst_CNKS = 'CNKS ' + filename inst_RETR = 'RETR ' + filename inst_UPDG = 'UPDG ' inst_NOOP = 'NOOP' inst_CACHEDATA = 'CACHEDATA ' ###### DECIDING WHICH CHUNKS TO DOWNLOAD FROM CACHES: TIME 0 ###### #if DEBUG_FULL: ## available_chunks = [0]*len(self.clients) # available_chunks[i] = cache i's availble chunks rates = [0]*len(self.clients) # rates[i] = cache i's offered rate union_chunks = [] # union of all available indices for i in range(len(self.clients)): client = self.clients[i] client.put_instruction(inst_CNKS) return_str = client.get_response().split('&') if return_str[0] == '': available_chunks[i] = [] else: available_chunks[i] = map(str, return_str[0].split('%')) for j in range(len(available_chunks[i])): available_chunks[i][j] = available_chunks[i][j].zfill(2) rates[i] = int(return_str[1]) union_chunks = list( set(union_chunks) | set(available_chunks[i]) ) ## index assignment here # Assign chunks to cache using cache_chunks_to_request. print '[user.py] Cache returned rates ', rates print '[user.py] Cache returned available chunks', available_chunks assigned_chunks = cache_chunks_to_request(available_chunks, rates, code_param_n, code_param_k) print '[user.py] Assigned chunks:', assigned_chunks effective_rates = [0]*len(rates) for i in range(len(rates)): effective_rates[i] = len(assigned_chunks[i]) print '[user.py] effective rates: ', effective_rates chosen_chunks = [j for i in assigned_chunks for j in i] print '[user.py] chosen chunks: ', chosen_chunks flag_deficit = int(sum(effective_rates) < code_param_k) # True if user needs more rate from caches list_of_cache_requests = [] # request assigned chunks for i in range(len(self.clients)): client = self.clients[i] client_ip_address = client.address[0] + '@' + str(client.address[1]) print '[user.py] Server_request (to cache) = "' , assigned_chunks[i] , '"' client_request_string = '%'.join(assigned_chunks[i]) + '&1' print "[user.py] [Client " + str(i) + "] flag_deficit: ", flag_deficit, \ ", Assigned chunks: ", assigned_chunks[i], \ ", Request string: ", client_request_string if DEBUG_FULL: pdb.set_trace() cachedata_request_string = client_request_string.replace('&1','&' + client_ip_address) list_of_cache_requests.append(filename+ '.' + cachedata_request_string) client.put_instruction(inst_UPDG + str(flag_deficit)) client.put_instruction(inst_RETR + '.' + client_request_string) if False: # WHY IS THIS NOT WORKING? if not assigned_chunks[i]: pass #client.put_instruction(inst_NOOP) else: client.put_instruction(inst_RETR + '.' + client_request_string) ###### DECIDING CHUNKS THAT HAVE TO BE DOWNLOADED FROM CACHE: TIME 0 ###### # Before CACHE_DOWNLOAD_DURATION, also start requesting chunks from server. server_request = [] server_request_2 = [] #cdrs = '_' .join(list_of_cache_requests) #cache data request string. Used for parsing inside of server.py's ftp_CACHEDATA if frame_number < num_frames: size_of_chunks = vlen_items[2] else: size_of_chunks = vlen_items[3] #cdrs = cdrs + '?' + str(size_of_chunks) #cdrs = cdrs + '_' + self.user_name chosen_chunks = list(chosen_chunks) num_chunks_rx_predicted = len(chosen_chunks) server_request = chunks_to_request(chosen_chunks, range(0, code_param_n), code_param_k - num_chunks_rx_predicted) num_of_chks_from_server = len(server_request) ## #UPLOAD SERVER CHUNKS TO THE GUI server_ip_address = self.server_client.address[0] + '@' + str(self.server_client.address[1]) server_chunk_string = '%' .join(server_request) srs = filename + '.' + server_chunk_string + '&' + server_ip_address list_of_cache_requests.insert(0,srs) #END UPLOAD SERVER CHUNKS TO THE GUI ## cdrs = '_' .join(list_of_cache_requests) cdrs = cdrs + '?' + str(size_of_chunks) cdrs = cdrs + '_' + self.user_name if num_of_chks_from_server == 0: self.server_client.put_instruction(inst_CACHEDATA + cdrs) self.server_client.put_instruction(inst_NOOP) print '[user.py] Caches handling code_param_k chunks, so no request to server. Sending a NOOP' else: print '[user.py] Server_request (to server) = "' , server_request , '"' server_request_string = '%'.join(server_request) + '&1' #if DEBUG_FULL: #pdb.set_trace() self.server_client.put_instruction(inst_CACHEDATA + cdrs) self.server_client.put_instruction(inst_RETR + '.' + server_request_string) if(DEBUGGING_MSG): print "[user.py] Requesting from server: ", server_request, ", Request string: ", server_request_string print "[user.py] Sending to server's ftp_CACHEDATA: ", inst_CACHEDATA + cdrs #update_server_load(tracker_address, video_name, num_of_chks_from_server) sleep(CACHE_DOWNLOAD_DURATION) ###### STOPPING CACHE DOWNLOADS: TIME 8 (CACHE_DOWNLOAD_DURATION) ###### # immediately stop cache downloads. for client in self.clients: try: ## client.client.abort() except: #I think should be EOFError #e = sys.exc_info()[0] ## print sys.exc_info()[0] print "[user.py] Cache connections suddenly aborted. Stopping all download." self.clients.remove(client) return print "[user.py] Cache connections aborted for frame %d" % (frame_number) ###### REQUEST ADDITIONAL CHUNKS FROM SERVER: TIME 8 (CACHE_DOWNLOAD_DURATION) ###### # Request from server remaining chunks missing # Look up the download directory and count the downloaded chunks chunk_nums_rx = chunk_nums_in_frame_dir(folder_name) if (DEBUGGING_MSG): print "%d chunks received so far for frame %d: " % (len(chunk_nums_rx), frame_number) print chunk_nums_rx # Add the chunks that have already been requested from server chunk_nums_rx = list (set(chunk_nums_in_frame_dir(folder_name)) | set(server_request)) print "[user.py] chunk_nums_rx", chunk_nums_rx addtl_server_request = [] num_chunks_rx = len(chunk_nums_rx) if (num_chunks_rx >= code_param_k): print "[user.py] No additional chunks to download from the server. Sending a NOOP" self.server_client.put_instruction(inst_NOOP) else: addtl_server_request = chunks_to_request(chunk_nums_rx, range(0, code_param_n), code_param_k - num_chunks_rx) print "[user.py] addtl_server_requests", addtl_server_request if addtl_server_request: addtl_server_request_string = '%'.join(addtl_server_request) + '&1' # The last digit '1' means 'I am user' # server should always be set with flag_deficit = 0 (has all chunks) self.server_client.put_instruction(inst_RETR + '.' + addtl_server_request_string) if(DEBUGGING_MSG): print "[user.py] Requesting from server: ", addtl_server_request elif (DEBUGGING_MSG): print "No unique chunks from server requested." ###### WAIT FOR CHUNKS FROM SERVER TO FINISH DOWNLOADING: TIME 10 ###### sleep(SERVER_DOWNLOAD_DURATION) if (DEBUGGING_MSG): print "[user.py] Waiting to receive all elements from server." if frame_number > start_frame and (server_request or addtl_server_request) and VLC_PLAYER_USE: # Need to pause it! self.VLC_pause_video() if server_request: resp_RETR = self.server_client.get_response() parsed_form = parse_chunks(resp_RETR) fname, framenum, chunks, user_or_cache = parsed_form print "[user.py] Downloaded chunks from server: ", chunks if addtl_server_request: resp_RETR = self.server_client.get_response() parsed_form = parse_chunks(resp_RETR) fname, framenum, chunks, user_or_cache = parsed_form print "[user.py] Downloaded chunks from server: ", chunks # Now play it if frame_number > start_frame and (server_request or addtl_server_request) and VLC_PLAYER_USE: self.VLC_pause_video() chunk_nums = chunk_nums_in_frame_dir(folder_name) num_chunks_rx = len(chunk_nums) if num_chunks_rx >= code_param_k: print "[user.py] Received", code_param_k, "packets, that is enough." else: print "[user.py] Did not receive", code_param_k, "packets for this frame, received: ", num_chunks_rx # abort the connection to the server self.server_client.client.abort() # put together chunks into single frame; then concatenate onto original file. print 'about to decode...' chunksList = chunk_files_in_frame_dir(folder_name) if frame_number != start_frame: print 'size of base file:', os.path.getsize('video-' + video_name + '/' + base_file_name) print 'trying to decode' filefec.decode_from_files(base_file, chunksList) print 'decoded. Size of base file =', os.path.getsize('video-' + video_name + '/' + base_file_name) if frame_number == 1 and VLC_PLAYER_USE: self.VLC_empty_list() self.VLC_start_video(base_file_full_path) if USER_TOPOLOGY_UPDATE: if choke_state == 0: # Normal state print '[user.py] Normal state : ', choke_ct choke_ct += 1 if choke_ct == T_choke: choke_ct = 0 if len(not_connected_caches) == 0: pass else: # Add a new cache temporarily new_cache_index = random.sample(range(len(not_connected_caches)), 1) if new_cache_index >= 0: new_cache = not_connected_caches[new_cache_index[0]] self.clients.append(new_cache) connected_caches.append(new_cache) not_connected_caches.remove(new_cache) print '[user.py] Topology Update : Temporarily added ', new_cache.address choke_state = 1 # Now, move to transitional state choke_ct = 0 print '[user.py] Topology Update : Now the state is changed to overhead staet' #print '[user.py]', connected_caches, not_connected_caches, self.clients print '[user.py -debug] connected caches', self.clients for c in range(len(self.clients)): print '[user.py -debug] cache addressaddress', self.clients[c].address elif choke_state == 1: # Overhead state print '[user.py] Overhead state : ', choke_ct choke_ct += 1 if choke_ct == T_choke2: # Temporary period to spend with temporarily added node rate_vector = [0] * len(self.clients) p_vector = [0] * len(self.clients) for i in range(len(self.clients)): rate_vector[i] = len(assigned_chunks[i]) p_vector[i] = math.exp( -eps_choke * rate_vector[i]) p_sum = sum(p_vector) for i in range(len(self.clients)): p_vector[i] /= p_sum cdf = [(0,0)] * len(self.clients) cdf[0] = (0, 0) for i in range(1, len(self.clients)): cdf[i] = (i, cdf[i-1][1] + p_vector[i-1]) print '[user.py] cdf :', cdf client_index = max(i for r in [random.random()] for i,c in cdf if c <= r) # http://stackoverflow.com/questions/4265988/generate-random-numbers-with-a-given-numerical-distribution removed_cache = self.clients[client_index] #removed_cache.put_instruction('QUIT') self.clients.remove(removed_cache) connected_caches.remove(removed_cache) not_connected_caches.append(removed_cache) print '[user.py] Topology Update : ', removed_cache.address, 'is chocked.' choke_state = 0 # Now, move to normal state choke_ct = 0 def disconnect(self, tracker_address, video_name, user_name): for client in self.clients: client.put_instruction('QUIT') for client in self.not_connected_caches: client.put_instruction('QUIT') self.server_client.put_instruction('QUIT') print "[user.py] Closed all connections." ## my_ip = user_name my_port = 0 my_video_name = video_name deregister_to_tracker_as_user(tracker_address, my_ip, my_port, video_name) #tracker_address, user_name, 0, video_name #deregister_to_tracker_as_cache(tracker_address) #for debug - chen self.info_thread.flag = False alarm(1) #wait 1 second before running alarm self.info_thread.join() alarm(0) print "[user.py] BYE" sys.stdout.flush()