def play(self, return_status=True):
print 'master service: in play'
success_response = utils.format_client_response(\
True, PLAY, {}, \
client_req_id=self._client_req_id)
# Calls forward then play again
if self._current_song == None:
self.current_offset = 0
if len(self._playlist_queue) > 0:
self.forward(return_status=False, play=False)
self.play(return_status=return_status)
return
# calculate approximate timeout for replica response
delay_buffer = 0
total_max_delay = 0
for ip in self._replicas:
total_max_delay += self._latency_by_ip[ip][2]
delay_buffer = int(2 * total_max_delay)
# global start time that all replicas must agree on
start_time = \
int(round(time.time() * MICROSECONDS)) + delay_buffer + EXTRA_BUFFER
for replica_ip in self._replicas:
local_start = start_time + int(
self._clock_difference_by_ip[replica_ip][0])
# each rpc runs on its own thread; send play command for local start time
# Note that the current song can be None in which case replica
# will stop playing current song
req_data = { 'song_hash' : self._current_song, \
'offset': self._current_offset, \
'start_time': local_start }
r = RPC(self, PLAY, url='http://' + replica_ip + PLAY_URL, \
ip=replica_ip, data=req_data)
r.start()
# TODO: same as pause to do a better ack-check-wait
time.sleep(float(2 * delay_buffer + 2 * EXTRA_BUFFER) / MICROSECONDS)
if self.rpc_response_acks >= 1: # check for acks
self._playing = True
if return_status:
self._status_queue.put(utils.format_client_response(\
True, PLAY, {}, \
client_req_id=self._client_req_id))
elif return_status:
self._status_queue.put(utils.format_client_response(\
False, PLAY, {}, \
client_req_id=self._client_req_id))
def pause(self):
total_max_delay = 0
for ip in self._replicas:
total_max_delay += self._latency_by_ip[ip][2]
delay_buffer = int(2 * total_max_delay)
# set stop time in same fashion as start time
stop_time = int(round(time.time() * MICROSECONDS)) + delay_buffer
for ip in self._replicas:
local_stop = stop_time + int(self._clock_difference_by_ip[ip][0])
r = RPC(self, PAUSE, url='http://' + ip + STOP_URL, \
ip=ip, data={"stop_time":local_stop})
r.start()
time.sleep(float(2 * delay_buffer + 2 * EXTRA_BUFFER) / MICROSECONDS)
# decide how far into song we are based on maximum offset from replicas
max_offset = 0
for offset in self.rpc_offsets:
if offset > self._current_offset:
self._current_offset = offset
# TODO: currently assume success; could count number of offset responses
self._playing = False
self._status_queue.put(utils.format_client_response(\
True, PAUSE, {}, \
client_req_id=self._client_req_id))
def load_song(self, song_hash):
print "master client: load song"
self.inc_client_req_id()
command_info = {
'command': LOAD,
'master_ip': self._ip,
'params': {
'song_hash': song_hash
},
'client_req_id': self._client_req_id
}
if request.method == 'GET':
if os.path.exists(utils.get_music_path(song_hash)):
self._command_queue.put(command_info)
return self.wait_on_master_music_service(LOAD)
# song doesn't exist on master, get the song from the client
else:
return utils.serialize_response(
utils.format_client_response(
False,
LOAD, {},
msg='Master does not have requested song'))
elif request.method == 'POST':
data = utils.unserialize_response(request.get_data())
with open(utils.get_music_path(song_hash), 'w') as f:
f.write(data['song_bytes'])
self._command_queue.put(command_info)
return self.wait_on_master_music_service(LOAD)
def execute_command(self, command):
print 'master client executing ' + command
self.inc_client_req_id()
if command in [PLAY, PAUSE, FORWARD, BACKWARD]:
command_info = {'command':command, 'master_ip': self._ip, 'params':{}, 'client_req_id': self._client_req_id}
self._command_queue.put(command_info)
return self.wait_on_master_music_service(command)
else:
return utils.serialize_response(utils.format_client_response(True, command, {}))
def backward(self):
self._current_offset = 0
# Play if song was previously playing
if self._playing:
self.play()
else:
self._status_queue.put(utils.format_client_response(\
True, BACKWARD, {}, \
client_req_id=self._client_req_id))
def load_song(self, params):
song_hash = params['song_hash']
# Check with replicas to see which have song
rpc_data = {}
self.exponential_backoff(rpc_data, CHECK, \
CHECK_URL + '/' + song_hash, REPLICA_ACK_TIMEOUT)
# warn replicas that we are about to load a song, so might miss some
# heartbeats
for replica_ip in self._replicas:
r = RPC(self, PRELOAD, url='http://' + replica_ip + PRELOAD_URL, \
ip=replica_ip, data={})
r.start()
# Loads songs to those who don't have it
rpc_data = None
while (not self.rpc_not_loaded_ips.empty()):
replica_ip = self.rpc_not_loaded_ips.get(block=False)
replica_url = \
'http://' + replica_ip + LOAD_URL + "/" + song_hash
if rpc_data == None:
with open(utils.get_music_path(song_hash), 'r') as f:
rpc_data = {'song_bytes': f.read()}
r = RPC(self, LOAD, url=replica_url, ip=replica_ip, data=rpc_data)
r.start()
print "sent load rpc"
if self.load_timeout():
self._status_queue.put(utils.format_client_response(\
False, LOAD, {}, \
msg='Timeout on song load', \
client_req_id=self._client_req_id))
else:
self._status_queue.put(utils.format_client_response(\
True, LOAD, {}, \
client_req_id=self._client_req_id))
# heartbeat, then notify that we are done loading so that normal
# failover checking can continue
self.heartbeat_all()
for replica_ip in self._replicas:
r = RPC(self, POSTLOAD, url='http://' + replica_ip + POSTLOAD_URL, \
ip=replica_ip, data={})
r.start()
def enqueue_song(self, song_hash):
print 'master client: enqueue song'
self.inc_client_req_id()
print 'in enqueue song client master'
command_info = {'command':ENQUEUE, 'master_ip': self._ip, 'params':{'song_hash':song_hash}, 'client_req_id': self._client_req_id}
if os.path.exists(utils.get_music_path(song_hash)):
# verify song exists on >= f+1 replicas and in their playlist
# queues
self._command_queue.put(command_info)
return self.wait_on_master_music_service(ENQUEUE)
# song doesn't exist on master, get the song from the client
else:
return utils.serialize_response(utils.format_client_response(False, ENQUEUE, {}, msg='Requested song to enqueue does not exist'))
def forward(self, return_status=True, play=True):
song = None
success_response = utils.format_client_response(\
True, FORWARD, {}, \
client_req_id=self._client_req_id)
# No song in future and no song currently playing, nothing to do.
if len(self._playlist_queue) == 0 and self._current_song == None:
if return_status:
self._status_queue.put(success_response)
return
# After a forward command we are always at the start of a song
self._current_offset = 0
# No songs to play anymore
if len(self._playlist_queue) == 0:
print "forward: no songs in playlist"
self._current_song = None
with open(PLAYLIST_STATE_FILE, 'w') as f:
data = utils.format_playlist_state(self._playlist_queue,
self._current_song)
f.write(data)
# Pop out a song to play
else:
print "forward: popping song"
self._current_song = self._playlist_queue.popleft()
with open(PLAYLIST_STATE_FILE, 'w') as f:
data = utils.format_playlist_state(self._playlist_queue,
self._current_song)
f.write(data)
hashed_post_playlist = utils.hash_string(
pickle.dumps(self._playlist_queue))
# Synchronizes dequeue operation across all replicas (for master recovery)
rpc_data = {'hashed_post_playlist': hashed_post_playlist, \
'current_song' : self._current_song, \
'time': time.time() }
# Try indefinitely until we get at least f+1 responses
# Guaranteed RPC won't add to queue since new command_epoch prevents
# Holding mutexes just in case
self.exponential_backoff(rpc_data, DEQUEUE, \
DEQUEUE_URL, \
REPLICA_ACK_TIMEOUT)
# Start playing the next song
# (if current_song == None then will just stop playing music)
if play:
self.play(False)
if return_status:
self._status_queue.put(success_response)
def execute_command(self, command):
print 'master client executing ' + command
self.inc_client_req_id()
if command in [PLAY, PAUSE, FORWARD, BACKWARD]:
command_info = {
'command': command,
'master_ip': self._ip,
'params': {},
'client_req_id': self._client_req_id
}
self._command_queue.put(command_info)
return self.wait_on_master_music_service(command)
else:
return utils.serialize_response(
utils.format_client_response(True, command, {}))
def load_song(self, song_hash):
print "master client: load song"
self.inc_client_req_id()
command_info = {'command':LOAD, 'master_ip': self._ip, 'params':{'song_hash':song_hash}, 'client_req_id': self._client_req_id}
if request.method == 'GET':
if os.path.exists(utils.get_music_path(song_hash)):
self._command_queue.put(command_info)
return self.wait_on_master_music_service(LOAD)
# song doesn't exist on master, get the song from the client
else:
return utils.serialize_response(utils.format_client_response(False, LOAD, {}, msg='Master does not have requested song'))
elif request.method == 'POST':
data = utils.unserialize_response(request.get_data())
with open(utils.get_music_path(song_hash), 'w') as f:
f.write(data['song_bytes'])
self._command_queue.put(command_info)
return self.wait_on_master_music_service(LOAD)
def wait_on_master_music_service(self, command):
status = None
i = 0
while(True):
if (i == 500):
break
try:
status = self._status_queue.get(False)
except Queue.Empty:
time.sleep(CLIENT_TIMEOUT / 500.0)
i += 1
else:
if self._client_req_id == status['client_req_id']:
break
if status == None:
status = utils.format_client_response(False, command, {}, msg='timeout from master', client_req_id=self._client_req_id)
return utils.serialize_response(status)
def enqueue_song(self, params):
song_hash = params['song_hash']
self.load_song(params)
self._playlist_queue.append(song_hash)
with open(PLAYLIST_STATE_FILE, 'w') as f:
data = utils.format_playlist_state(self._playlist_queue,
self._current_song)
f.write(data)
hashed_post_playlist = utils.hash_string(
pickle.dumps(self._playlist_queue))
rpc_data = {'current_song': self._current_song, \
'hashed_post_playlist': hashed_post_playlist, \
'time': time.time() }
self.exponential_backoff(rpc_data, ENQUEUE, \
ENQUEUE_URL + '/' + song_hash, \
REPLICA_ACK_TIMEOUT)
self._status_queue.put(utils.format_client_response(\
True, ENQUEUE, {}, \
client_req_id=self._client_req_id))
def wait_on_master_music_service(self, command):
status = None
i = 0
while (True):
if (i == 500):
break
try:
status = self._status_queue.get(False)
except Queue.Empty:
time.sleep(CLIENT_TIMEOUT / 500.0)
i += 1
else:
if self._client_req_id == status['client_req_id']:
break
if status == None:
status = utils.format_client_response(
False,
command, {},
msg='timeout from master',
client_req_id=self._client_req_id)
return utils.serialize_response(status)
def enqueue_song(self, song_hash):
print 'master client: enqueue song'
self.inc_client_req_id()
print 'in enqueue song client master'
command_info = {
'command': ENQUEUE,
'master_ip': self._ip,
'params': {
'song_hash': song_hash
},
'client_req_id': self._client_req_id
}
if os.path.exists(utils.get_music_path(song_hash)):
# verify song exists on >= f+1 replicas and in their playlist
# queues
self._command_queue.put(command_info)
return self.wait_on_master_music_service(ENQUEUE)
# song doesn't exist on master, get the song from the client
else:
return utils.serialize_response(
utils.format_client_response(
False,
ENQUEUE, {},
msg='Requested song to enqueue does not exist'))
