def recover_replica(self): data = utils.unserialize_response(request.get_data()) replica_song_hashes = data['song_hashes'] with open(PLAYLIST_STATE_FILE, 'r') as f: data = utils.unserialize_response(f.read()) print data master_queue = data['playlist'] current_song = data['current_song'] missing_songs = {} print master_queue for song_hash in master_queue: print 'song_hash ' + str(song_hash) music_path = utils.get_music_path(song_hash) if (os.path.exists(music_path)) and ( replica_song_hashes.count(song_hash + EXT) == 0): with open(music_path, 'r') as f: song_bytes = f.read() missing_songs[song_hash + EXT] = song_bytes if current_song != None: music_path = utils.get_music_path(current_song) if (os.path.exists(current_song + EXT)) and ( replica_song_hashes.count(current_song + EXT) == 0): with open(music_path, 'r') as f: song_bytes = f.read() missing_songs[current_song + EXT] = song_bytes resp = utils.format_rpc_response( True, RECOVER, { 'songs': missing_songs, 'master_queue': master_queue, 'current_song': current_song }) return utils.serialize_response(resp)
def load_song(self, song_hash): print "master client: load song" self.inc_client_req_id() command_info = { 'command': LOAD, 'master_ip': self._ip, 'params': { 'song_hash': song_hash }, 'client_req_id': self._client_req_id } if request.method == 'GET': if os.path.exists(utils.get_music_path(song_hash)): self._command_queue.put(command_info) return self.wait_on_master_music_service(LOAD) # song doesn't exist on master, get the song from the client else: return utils.serialize_response( utils.format_client_response( False, LOAD, {}, msg='Master does not have requested song')) elif request.method == 'POST': data = utils.unserialize_response(request.get_data()) with open(utils.get_music_path(song_hash), 'w') as f: f.write(data['song_bytes']) self._command_queue.put(command_info) return self.wait_on_master_music_service(LOAD)
def recover_replica(self): data = utils.unserialize_response(request.get_data()) replica_song_hashes = data['song_hashes'] with open(PLAYLIST_STATE_FILE, 'r') as f: data = utils.unserialize_response(f.read()) print data master_queue = data['playlist'] current_song = data['current_song'] missing_songs = {} print master_queue for song_hash in master_queue: print 'song_hash ' + str(song_hash) music_path = utils.get_music_path(song_hash) if (os.path.exists(music_path)) and (replica_song_hashes.count(song_hash + EXT) == 0): with open(music_path, 'r') as f: song_bytes = f.read() missing_songs[song_hash + EXT] = song_bytes if current_song != None: music_path = utils.get_music_path(current_song) if (os.path.exists(current_song + EXT)) and (replica_song_hashes.count(current_song + EXT) == 0): with open(music_path, 'r') as f: song_bytes = f.read() missing_songs[current_song + EXT] = song_bytes resp = utils.format_rpc_response(True, RECOVER, {'songs': missing_songs, 'master_queue': master_queue, 'current_song': current_song}) return utils.serialize_response(resp)
def load_song(self, song_hash): print "master client: load song" self.inc_client_req_id() command_info = {'command':LOAD, 'master_ip': self._ip, 'params':{'song_hash':song_hash}, 'client_req_id': self._client_req_id} if request.method == 'GET': if os.path.exists(utils.get_music_path(song_hash)): self._command_queue.put(command_info) return self.wait_on_master_music_service(LOAD) # song doesn't exist on master, get the song from the client else: return utils.serialize_response(utils.format_client_response(False, LOAD, {}, msg='Master does not have requested song')) elif request.method == 'POST': data = utils.unserialize_response(request.get_data()) with open(utils.get_music_path(song_hash), 'w') as f: f.write(data['song_bytes']) self._command_queue.put(command_info) return self.wait_on_master_music_service(LOAD)
def enqueue_song(self, song_hash): print 'master client: enqueue song' self.inc_client_req_id() print 'in enqueue song client master' command_info = {'command':ENQUEUE, 'master_ip': self._ip, 'params':{'song_hash':song_hash}, 'client_req_id': self._client_req_id} if os.path.exists(utils.get_music_path(song_hash)): # verify song exists on >= f+1 replicas and in their playlist # queues self._command_queue.put(command_info) return self.wait_on_master_music_service(ENQUEUE) # song doesn't exist on master, get the song from the client else: return utils.serialize_response(utils.format_client_response(False, ENQUEUE, {}, msg='Requested song to enqueue does not exist'))
def enqueue_song(self, song_hash): self._last_hb_ts = time.time() * MICROSECONDS content = utils.unserialize_response(request.get_data()) command_epoch = content['command_epoch'] master_post_hash = content['hashed_post_playlist'] master_current_song = content['current_song'] failover_resp = utils.format_rpc_response(False, ENQUEUE, {}, \ msg='Replica in recovery mode', \ command_epoch=command_epoch) if self._in_recovery: return utils.serialize_response(failover_resp) print "In Enqueue" print "Enqueue: " + str(self._current_song) replica_pre_hash = utils.hash_string(pickle.dumps( self._playlist_queue)) if replica_pre_hash == master_post_hash and self._current_song == master_current_song: print "Already Performed Operation in Enqueue" repeat_resp = utils.format_rpc_response(False, ENQUEUE, {}, \ msg='Already performed operation', \ command_epoch=command_epoch) return utils.serialize_response(repeat_resp) # Do enqueue, check for failover mode song_not_exist = not os.path.exists(utils.get_music_path(song_hash)) self._playlist_queue.append(song_hash) replica_post_hash = utils.hash_string( pickle.dumps(self._playlist_queue)) inconsistent_queue = master_post_hash != replica_post_hash or \ master_current_song != self._current_song print "queue hashes match: " + str( master_post_hash == replica_post_hash) print "current song matches: " + str( master_current_song) + " == " + str(self._current_song) replica_failover = song_not_exist or inconsistent_queue if replica_failover: self._in_recovery = True return utils.serialize_response(failover_resp) master_time = content['time'] self._master_timestamp = master_time resp = utils.format_rpc_response(True, ENQUEUE, {'enqueued': True}, \ command_epoch=command_epoch) print str(resp) return utils.serialize_response(resp)
def load_song(self, params): song_hash = params['song_hash'] # Check with replicas to see which have song rpc_data = {} self.exponential_backoff(rpc_data, CHECK, \ CHECK_URL + '/' + song_hash, REPLICA_ACK_TIMEOUT) # warn replicas that we are about to load a song, so might miss some # heartbeats for replica_ip in self._replicas: r = RPC(self, PRELOAD, url='http://' + replica_ip + PRELOAD_URL, \ ip=replica_ip, data={}) r.start() # Loads songs to those who don't have it rpc_data = None while (not self.rpc_not_loaded_ips.empty()): replica_ip = self.rpc_not_loaded_ips.get(block=False) replica_url = \ 'http://' + replica_ip + LOAD_URL + "/" + song_hash if rpc_data == None: with open(utils.get_music_path(song_hash), 'r') as f: rpc_data = {'song_bytes': f.read()} r = RPC(self, LOAD, url=replica_url, ip=replica_ip, data=rpc_data) r.start() print "sent load rpc" if self.load_timeout(): self._status_queue.put(utils.format_client_response(\ False, LOAD, {}, \ msg='Timeout on song load', \ client_req_id=self._client_req_id)) else: self._status_queue.put(utils.format_client_response(\ True, LOAD, {}, \ client_req_id=self._client_req_id)) # heartbeat, then notify that we are done loading so that normal # failover checking can continue self.heartbeat_all() for replica_ip in self._replicas: r = RPC(self, POSTLOAD, url='http://' + replica_ip + POSTLOAD_URL, \ ip=replica_ip, data={}) r.start()
def load_song(self, song_hash): content = utils.unserialize_response(request.get_data()) command_epoch = content['command_epoch'] song_bytes = content['song_bytes'] if self._in_recovery: failover_resp = utils.format_rpc_response(False, LOAD, {}, \ msg='Replica in recovery mode', \ command_epoch=command_epoch) self._last_hb_ts = time.time() * MICROSECONDS return utils.serialize_response(failover_resp) try: with open(utils.get_music_path(song_hash), 'w') as f: f.write(song_bytes) except Exception: resp = utils.format_rpc_response(False, LOAD, {}, \ msg='Error occured in writing to replica', \ command_epoch=command_epoch) else: self._song_hashes.add(song_hash) resp = utils.format_rpc_response(True, LOAD, \ {'has_song': True, 'ip':self._ip}, \ command_epoch=command_epoch) self._last_hb_ts = time.time() * MICROSECONDS return utils.serialize_response(resp)
def enqueue_song(self, song_hash): print 'master client: enqueue song' self.inc_client_req_id() print 'in enqueue song client master' command_info = { 'command': ENQUEUE, 'master_ip': self._ip, 'params': { 'song_hash': song_hash }, 'client_req_id': self._client_req_id } if os.path.exists(utils.get_music_path(song_hash)): # verify song exists on >= f+1 replicas and in their playlist # queues self._command_queue.put(command_info) return self.wait_on_master_music_service(ENQUEUE) # song doesn't exist on master, get the song from the client else: return utils.serialize_response( utils.format_client_response( False, ENQUEUE, {}, msg='Requested song to enqueue does not exist'))
def play(self): self._last_hb_ts = time.time() * MICROSECONDS # Parse payload content = utils.unserialize_response(request.get_data()) command_epoch = content['command_epoch'] start_time_microsec = content['start_time'] # Not sure when it would be -1 but double check assert (start_time_microsec >= 0) master_offset = 0 if 'offset' in content: master_offset = int(content['offset']) # May be None, need to call .stop() then song_hash = content['song_hash'] failover_mode_resp = utils.format_rpc_response(\ False, PLAY, {}, \ msg='Replica in recovery mode', \ command_epoch=command_epoch) # Case 1: Failover Mode already # Short circuit crucial to not starve failover_service process on # pygame_mixer object # Case 2: Song hash doesn't exist if self._in_recovery or (song_hash != None and not \ os.path.exists(utils.get_music_path(song_hash))): self._in_recovery = True return utils.serialize_response(failover_mode_resp) # Could be in recovery mode at this point, wait for it to finish # before moving on (True) with self._pygame_lock: replica_offset = int(round(pygame.mixer.music.get_pos())) offset_diff = master_offset - replica_offset # Ideally if pygame.mixer.music.play(offset) works then these would not be # errors but alas such is life and we must go into recovery if master_offset > 0: # Case 3: Unpause command, but songs do not match # Case 4: Unpause command, but master's offset is way in future # This should not happen since master determines # offset as max of replica offsets if self._current_song != song_hash or \ abs(offset_diff) > MAX_OFFSET_DIF: self._in_recovery = True return utils.serialize_response(failover_mode_resp) elif song_hash != None: print "Loaded song" pygame.mixer.music.stop() pygame.mixer.music.load(utils.get_music_path(song_hash)) # Adjust start_time_microsec to account for offset difference if master_offset > 0: start_time_microsec = \ start_time_microsec - (offset_diff*MILLISECONDS) # wait until start time, then play curr_replica_microsec = int(round(time.time() * MICROSECONDS)) print song_hash print master_offset while (curr_replica_microsec + ALLOWED_REPLICA_BUFFER < start_time_microsec): curr_replica_microsec = int(round(time.time() * MICROSECONDS)) print "Song hash: " + str(song_hash) if song_hash == None: pygame.mixer.music.stop() elif master_offset == 0: pygame.mixer.music.play(1) else: pygame.mixer.music.unpause() time.sleep(2) # necessary to allow mp3 thread to start? self._current_song = song_hash curr_replica_microsec = int(round(time.time() * MICROSECONDS)) resp = utils.format_rpc_response(True, PLAY, \ {'time': curr_replica_microsec }, \ command_epoch='command_epoch') print "Play: " + str(self._current_song) return utils.serialize_response(resp)