def play(self, return_status=True): print 'master service: in play' success_response = utils.format_client_response(\ True, PLAY, {}, \ client_req_id=self._client_req_id) # Calls forward then play again if self._current_song == None: self.current_offset = 0 if len(self._playlist_queue) > 0: self.forward(return_status=False, play=False) self.play(return_status=return_status) return # calculate approximate timeout for replica response delay_buffer = 0 total_max_delay = 0 for ip in self._replicas: total_max_delay += self._latency_by_ip[ip][2] delay_buffer = int(2 * total_max_delay) # global start time that all replicas must agree on start_time = \ int(round(time.time() * MICROSECONDS)) + delay_buffer + EXTRA_BUFFER for replica_ip in self._replicas: local_start = start_time + int( self._clock_difference_by_ip[replica_ip][0]) # each rpc runs on its own thread; send play command for local start time # Note that the current song can be None in which case replica # will stop playing current song req_data = { 'song_hash' : self._current_song, \ 'offset': self._current_offset, \ 'start_time': local_start } r = RPC(self, PLAY, url='http://' + replica_ip + PLAY_URL, \ ip=replica_ip, data=req_data) r.start() # TODO: same as pause to do a better ack-check-wait time.sleep(float(2 * delay_buffer + 2 * EXTRA_BUFFER) / MICROSECONDS) if self.rpc_response_acks >= 1: # check for acks self._playing = True if return_status: self._status_queue.put(utils.format_client_response(\ True, PLAY, {}, \ client_req_id=self._client_req_id)) elif return_status: self._status_queue.put(utils.format_client_response(\ False, PLAY, {}, \ client_req_id=self._client_req_id))
def pause(self): total_max_delay = 0 for ip in self._replicas: total_max_delay += self._latency_by_ip[ip][2] delay_buffer = int(2 * total_max_delay) # set stop time in same fashion as start time stop_time = int(round(time.time() * MICROSECONDS)) + delay_buffer for ip in self._replicas: local_stop = stop_time + int(self._clock_difference_by_ip[ip][0]) r = RPC(self, PAUSE, url='http://' + ip + STOP_URL, \ ip=ip, data={"stop_time":local_stop}) r.start() time.sleep(float(2 * delay_buffer + 2 * EXTRA_BUFFER) / MICROSECONDS) # decide how far into song we are based on maximum offset from replicas max_offset = 0 for offset in self.rpc_offsets: if offset > self._current_offset: self._current_offset = offset # TODO: currently assume success; could count number of offset responses self._playing = False self._status_queue.put(utils.format_client_response(\ True, PAUSE, {}, \ client_req_id=self._client_req_id))
def load_song(self, song_hash): print "master client: load song" self.inc_client_req_id() command_info = { 'command': LOAD, 'master_ip': self._ip, 'params': { 'song_hash': song_hash }, 'client_req_id': self._client_req_id } if request.method == 'GET': if os.path.exists(utils.get_music_path(song_hash)): self._command_queue.put(command_info) return self.wait_on_master_music_service(LOAD) # song doesn't exist on master, get the song from the client else: return utils.serialize_response( utils.format_client_response( False, LOAD, {}, msg='Master does not have requested song')) elif request.method == 'POST': data = utils.unserialize_response(request.get_data()) with open(utils.get_music_path(song_hash), 'w') as f: f.write(data['song_bytes']) self._command_queue.put(command_info) return self.wait_on_master_music_service(LOAD)
def execute_command(self, command): print 'master client executing ' + command self.inc_client_req_id() if command in [PLAY, PAUSE, FORWARD, BACKWARD]: command_info = {'command':command, 'master_ip': self._ip, 'params':{}, 'client_req_id': self._client_req_id} self._command_queue.put(command_info) return self.wait_on_master_music_service(command) else: return utils.serialize_response(utils.format_client_response(True, command, {}))
def backward(self): self._current_offset = 0 # Play if song was previously playing if self._playing: self.play() else: self._status_queue.put(utils.format_client_response(\ True, BACKWARD, {}, \ client_req_id=self._client_req_id))
def load_song(self, params): song_hash = params['song_hash'] # Check with replicas to see which have song rpc_data = {} self.exponential_backoff(rpc_data, CHECK, \ CHECK_URL + '/' + song_hash, REPLICA_ACK_TIMEOUT) # warn replicas that we are about to load a song, so might miss some # heartbeats for replica_ip in self._replicas: r = RPC(self, PRELOAD, url='http://' + replica_ip + PRELOAD_URL, \ ip=replica_ip, data={}) r.start() # Loads songs to those who don't have it rpc_data = None while (not self.rpc_not_loaded_ips.empty()): replica_ip = self.rpc_not_loaded_ips.get(block=False) replica_url = \ 'http://' + replica_ip + LOAD_URL + "/" + song_hash if rpc_data == None: with open(utils.get_music_path(song_hash), 'r') as f: rpc_data = {'song_bytes': f.read()} r = RPC(self, LOAD, url=replica_url, ip=replica_ip, data=rpc_data) r.start() print "sent load rpc" if self.load_timeout(): self._status_queue.put(utils.format_client_response(\ False, LOAD, {}, \ msg='Timeout on song load', \ client_req_id=self._client_req_id)) else: self._status_queue.put(utils.format_client_response(\ True, LOAD, {}, \ client_req_id=self._client_req_id)) # heartbeat, then notify that we are done loading so that normal # failover checking can continue self.heartbeat_all() for replica_ip in self._replicas: r = RPC(self, POSTLOAD, url='http://' + replica_ip + POSTLOAD_URL, \ ip=replica_ip, data={}) r.start()
def enqueue_song(self, song_hash): print 'master client: enqueue song' self.inc_client_req_id() print 'in enqueue song client master' command_info = {'command':ENQUEUE, 'master_ip': self._ip, 'params':{'song_hash':song_hash}, 'client_req_id': self._client_req_id} if os.path.exists(utils.get_music_path(song_hash)): # verify song exists on >= f+1 replicas and in their playlist # queues self._command_queue.put(command_info) return self.wait_on_master_music_service(ENQUEUE) # song doesn't exist on master, get the song from the client else: return utils.serialize_response(utils.format_client_response(False, ENQUEUE, {}, msg='Requested song to enqueue does not exist'))
def forward(self, return_status=True, play=True): song = None success_response = utils.format_client_response(\ True, FORWARD, {}, \ client_req_id=self._client_req_id) # No song in future and no song currently playing, nothing to do. if len(self._playlist_queue) == 0 and self._current_song == None: if return_status: self._status_queue.put(success_response) return # After a forward command we are always at the start of a song self._current_offset = 0 # No songs to play anymore if len(self._playlist_queue) == 0: print "forward: no songs in playlist" self._current_song = None with open(PLAYLIST_STATE_FILE, 'w') as f: data = utils.format_playlist_state(self._playlist_queue, self._current_song) f.write(data) # Pop out a song to play else: print "forward: popping song" self._current_song = self._playlist_queue.popleft() with open(PLAYLIST_STATE_FILE, 'w') as f: data = utils.format_playlist_state(self._playlist_queue, self._current_song) f.write(data) hashed_post_playlist = utils.hash_string( pickle.dumps(self._playlist_queue)) # Synchronizes dequeue operation across all replicas (for master recovery) rpc_data = {'hashed_post_playlist': hashed_post_playlist, \ 'current_song' : self._current_song, \ 'time': time.time() } # Try indefinitely until we get at least f+1 responses # Guaranteed RPC won't add to queue since new command_epoch prevents # Holding mutexes just in case self.exponential_backoff(rpc_data, DEQUEUE, \ DEQUEUE_URL, \ REPLICA_ACK_TIMEOUT) # Start playing the next song # (if current_song == None then will just stop playing music) if play: self.play(False) if return_status: self._status_queue.put(success_response)
def execute_command(self, command): print 'master client executing ' + command self.inc_client_req_id() if command in [PLAY, PAUSE, FORWARD, BACKWARD]: command_info = { 'command': command, 'master_ip': self._ip, 'params': {}, 'client_req_id': self._client_req_id } self._command_queue.put(command_info) return self.wait_on_master_music_service(command) else: return utils.serialize_response( utils.format_client_response(True, command, {}))
def load_song(self, song_hash): print "master client: load song" self.inc_client_req_id() command_info = {'command':LOAD, 'master_ip': self._ip, 'params':{'song_hash':song_hash}, 'client_req_id': self._client_req_id} if request.method == 'GET': if os.path.exists(utils.get_music_path(song_hash)): self._command_queue.put(command_info) return self.wait_on_master_music_service(LOAD) # song doesn't exist on master, get the song from the client else: return utils.serialize_response(utils.format_client_response(False, LOAD, {}, msg='Master does not have requested song')) elif request.method == 'POST': data = utils.unserialize_response(request.get_data()) with open(utils.get_music_path(song_hash), 'w') as f: f.write(data['song_bytes']) self._command_queue.put(command_info) return self.wait_on_master_music_service(LOAD)
def wait_on_master_music_service(self, command): status = None i = 0 while(True): if (i == 500): break try: status = self._status_queue.get(False) except Queue.Empty: time.sleep(CLIENT_TIMEOUT / 500.0) i += 1 else: if self._client_req_id == status['client_req_id']: break if status == None: status = utils.format_client_response(False, command, {}, msg='timeout from master', client_req_id=self._client_req_id) return utils.serialize_response(status)
def enqueue_song(self, params): song_hash = params['song_hash'] self.load_song(params) self._playlist_queue.append(song_hash) with open(PLAYLIST_STATE_FILE, 'w') as f: data = utils.format_playlist_state(self._playlist_queue, self._current_song) f.write(data) hashed_post_playlist = utils.hash_string( pickle.dumps(self._playlist_queue)) rpc_data = {'current_song': self._current_song, \ 'hashed_post_playlist': hashed_post_playlist, \ 'time': time.time() } self.exponential_backoff(rpc_data, ENQUEUE, \ ENQUEUE_URL + '/' + song_hash, \ REPLICA_ACK_TIMEOUT) self._status_queue.put(utils.format_client_response(\ True, ENQUEUE, {}, \ client_req_id=self._client_req_id))
def wait_on_master_music_service(self, command): status = None i = 0 while (True): if (i == 500): break try: status = self._status_queue.get(False) except Queue.Empty: time.sleep(CLIENT_TIMEOUT / 500.0) i += 1 else: if self._client_req_id == status['client_req_id']: break if status == None: status = utils.format_client_response( False, command, {}, msg='timeout from master', client_req_id=self._client_req_id) return utils.serialize_response(status)
def enqueue_song(self, song_hash): print 'master client: enqueue song' self.inc_client_req_id() print 'in enqueue song client master' command_info = { 'command': ENQUEUE, 'master_ip': self._ip, 'params': { 'song_hash': song_hash }, 'client_req_id': self._client_req_id } if os.path.exists(utils.get_music_path(song_hash)): # verify song exists on >= f+1 replicas and in their playlist # queues self._command_queue.put(command_info) return self.wait_on_master_music_service(ENQUEUE) # song doesn't exist on master, get the song from the client else: return utils.serialize_response( utils.format_client_response( False, ENQUEUE, {}, msg='Requested song to enqueue does not exist'))