示例#1
0
def backward():
    url = get_url(BACKWARD)
    try:
        r = urllib2.urlopen(url)
        print utils.unserialize_response(r.read())
    except Exception:
        print "Error in Backwarding Song"
示例#2
0
def play():
    url = get_url(PLAY)
    try:
        r = urllib2.urlopen(url)
        print utils.unserialize_response(r.read())
    except Exception:
        print "Error in Playing Song"
示例#3
0
def load(song_file):
    song_path = MUSIC_DIR + song_file
    m = hashlib.md5()
    assert(os.path.exists(song_path))
    with open(song_path, 'r') as f:
        song_bytes = f.read()
        song_hash = hashlib.sha224(song_bytes).hexdigest()
    url = get_url(LOAD) + "/" + song_hash
    try:
        r = urllib2.urlopen(url)
    except Exception:
        print "Error in Checking Song"
    else:
        master_response = utils.unserialize_response(r.read())
        has_file = master_response['success']
        if not has_file:
            try:
                req = urllib2.Request(url)
            except Exception:
                print "Error in Uploading Song"
            else:
                d = {'song_bytes': song_bytes}
                req.add_data(utils.serialize_response(d))
                r = urllib2.urlopen(req)
                master_response = utils.unserialize_response(r.read())
    
        if not master_response['success']:
            print master_response['msg']
        print master_response['client_req_id']
    return 'load'
    def recover_replica(self):
        data = utils.unserialize_response(request.get_data())
        replica_song_hashes = data['song_hashes']
        with open(PLAYLIST_STATE_FILE, 'r') as f:
            data = utils.unserialize_response(f.read())
            print data
            master_queue = data['playlist']
            current_song = data['current_song']
        missing_songs = {}

        print master_queue
        for song_hash in master_queue:
            print 'song_hash ' + str(song_hash)
            music_path = utils.get_music_path(song_hash)
            if (os.path.exists(music_path)) and (replica_song_hashes.count(song_hash + EXT) == 0):
                with open(music_path, 'r') as f:
                    song_bytes = f.read()
                missing_songs[song_hash + EXT] = song_bytes
        if current_song != None:
            music_path = utils.get_music_path(current_song)
            if (os.path.exists(current_song + EXT)) and (replica_song_hashes.count(current_song + EXT) == 0):
                with open(music_path, 'r') as f:
                    song_bytes = f.read()
                missing_songs[current_song + EXT] = song_bytes

        resp = utils.format_rpc_response(True, RECOVER, {'songs': missing_songs, 'master_queue': master_queue, 'current_song': current_song})
        return utils.serialize_response(resp)
示例#5
0
def load_song(song_path):
    m = hashlib.md5()
    assert (os.path.exists(song_path))
    with open(song_path, 'r') as f:
        song_bytes = f.read()
        song_hash = hashlib.sha224(song_bytes).hexdigest()
    url = get_url(LOAD) + "/" + song_hash
    try:
        r = urllib2.urlopen(url)
    except Exception:
        print "Error in Checking Song"
    else:
        master_response = utils.unserialize_response(r.read())
        has_file = master_response['success']
        if not has_file:
            try:
                req = urllib2.Request(url)
            except Exception:
                print "Error in Uploading Song"
            else:
                d = {'song_bytes': song_bytes}
                req.add_data(utils.serialize_response(d))
                r = urllib2.urlopen(req)
                master_response = utils.unserialize_response(r.read())

        if not master_response['success']:
            print master_response['msg']
        print master_response['client_req_id']
示例#6
0
def backward():
    url = get_url(BACKWARD)
    try:
        r = urllib2.urlopen(url)
        print utils.unserialize_response(r.read())
    except Exception:
        print "Error in Backwarding Song"
示例#7
0
    def recover_replica(self):
        data = utils.unserialize_response(request.get_data())
        replica_song_hashes = data['song_hashes']
        with open(PLAYLIST_STATE_FILE, 'r') as f:
            data = utils.unserialize_response(f.read())
            print data
            master_queue = data['playlist']
            current_song = data['current_song']
        missing_songs = {}

        print master_queue
        for song_hash in master_queue:
            print 'song_hash ' + str(song_hash)
            music_path = utils.get_music_path(song_hash)
            if (os.path.exists(music_path)) and (
                    replica_song_hashes.count(song_hash + EXT) == 0):
                with open(music_path, 'r') as f:
                    song_bytes = f.read()
                missing_songs[song_hash + EXT] = song_bytes
        if current_song != None:
            music_path = utils.get_music_path(current_song)
            if (os.path.exists(current_song + EXT)) and (
                    replica_song_hashes.count(current_song + EXT) == 0):
                with open(music_path, 'r') as f:
                    song_bytes = f.read()
                missing_songs[current_song + EXT] = song_bytes

        resp = utils.format_rpc_response(
            True, RECOVER, {
                'songs': missing_songs,
                'master_queue': master_queue,
                'current_song': current_song
            })
        return utils.serialize_response(resp)
示例#8
0
def play():
    url = get_url(PLAY)
    try:
        r = urllib2.urlopen(url)
        print utils.unserialize_response(r.read())
    except Exception:
        print "Error in Playing Song"
示例#9
0
def pause():
    url = get_url(PAUSE)
    print url
    try:
        r = urllib2.urlopen(url)
        print utils.unserialize_response(r.read())
    except Exception:
        print "Error in Pausing Song"
示例#10
0
def pause():
    url = get_url(PAUSE)
    print url
    try:
        r = urllib2.urlopen(url)
        print utils.unserialize_response(r.read())
    except Exception:
        print "Error in Pausing Song"
示例#11
0
 def load_song(self, song_hash):
     print "master client: load song"
     self.inc_client_req_id()
     command_info = {
         'command': LOAD,
         'master_ip': self._ip,
         'params': {
             'song_hash': song_hash
         },
         'client_req_id': self._client_req_id
     }
     if request.method == 'GET':
         if os.path.exists(utils.get_music_path(song_hash)):
             self._command_queue.put(command_info)
             return self.wait_on_master_music_service(LOAD)
         # song doesn't exist on master, get the song from the client
         else:
             return utils.serialize_response(
                 utils.format_client_response(
                     False,
                     LOAD, {},
                     msg='Master does not have requested song'))
     elif request.method == 'POST':
         data = utils.unserialize_response(request.get_data())
         with open(utils.get_music_path(song_hash), 'w') as f:
             f.write(data['song_bytes'])
         self._command_queue.put(command_info)
         return self.wait_on_master_music_service(LOAD)
示例#12
0
 def get_time(self):
     self._last_hb_ts = time.time() * MICROSECONDS
     content = utils.unserialize_response(request.get_data())
     command_epoch = content['command_epoch']
     if "term" in content and int(content["term"]) > self._master_term:
         self._master_term = int(content["term"])
     if "ip" in content and (self._master_ip == None
                             or self._master_ip != content["ip"]):
         self._master_ip = content["ip"]
     curr_time = time.time() * MICROSECONDS
     curr_micros = int(round(curr_time))
     if self._in_recovery:
         failover_resp = utils.format_rpc_response(False, HB, {}, \
                                              msg='Replica in recovery mode', \
                                              command_epoch=command_epoch)
         return utils.serialize_response(failover_resp)
     with self._pygame_lock:
         replica_playing = pygame.mixer.music.get_busy()
     # Song has finished playing
     if not replica_playing:
         self._current_song = None
     resp = utils.format_rpc_response(True, HB, \
                                      {'time' : curr_micros, \
                                       'replica_playing': replica_playing},
                                       command_epoch = command_epoch)
     return utils.serialize_response(resp)
示例#13
0
    def pause(self):
        self._last_hb_ts = time.time() * MICROSECONDS
        content = utils.unserialize_response(request.get_data())
        command_epoch = content['command_epoch']
        master_stop_micros = content['stop_time']
        if self._in_recovery:
            failover_resp = utils.format_rpc_response(False, PAUSE, {}, \
                                                 msg='Replica in recovery mode', \
                                                 command_epoch=command_epoch)
            return utils.serialize_response(failover_resp)

        # Wait till appointed stop time, allowed to be less precise than play
        # (if vs while loop)
        curr_replica_micros = int(round(time.time() * MICROSECONDS))
        if (curr_replica_micros < master_stop_micros):
            time.sleep((master_stop_micros - curr_replica_micros) /
                       float(MICROSECONDS))

        with self._pygame_lock:
            pygame.mixer.music.pause()

            # return offset from start of song
            replica_offset = int(round(pygame.mixer.music.get_pos()))
            replica_micros = int(round(time.time() * MICROSECONDS))
            resp = \
                utils.format_rpc_response(True, PAUSE, \
                                          {'time': replica_micros, \
                                           'offset': replica_offset}, \
                                           command_epoch=command_epoch)
            return utils.serialize_response(resp)
示例#14
0
def pause():
    url = get_url(PAUSE)
    try:
        r = urllib2.urlopen(url)
        resp = utils.unserialize_response(r.read())
    except Exception:
        resp = {"success": False, "msg": "Error in Pausing Song"}
    return resp
示例#15
0
def backward():
    url = get_url(BACKWARD)
    try:
        r = urllib2.urlopen(url)
        resp = utils.unserialize_response(r.read())
    except Exception:
        resp = {"success": False, "msg": "Error in Backwarding Song"}
    return resp
示例#16
0
def pause():
    url = get_url(PAUSE)
    try:
        r = urllib2.urlopen(url)
        resp = utils.unserialize_response(r.read())
    except Exception:
        resp = {"success": False, "msg": "Error in Pausing Song"}
    return resp
示例#17
0
def backward():
    url = get_url(BACKWARD)
    try:
        r = urllib2.urlopen(url)
        resp = utils.unserialize_response(r.read())
    except Exception:
        resp = {"success": False, "msg": "Error in Backwarding Song"}
    return resp
示例#18
0
def forward():
    global playlist
    url = get_url(FORWARD)
    try:
        r = urllib2.urlopen(url)
        resp = utils.unserialize_response(r.read())
        if len(playlist) != 0:
            playlist.pop(0)
    except Exception:
        resp = {"success": False, "msg": "Error in Forwarding Song"}
    return resp
示例#19
0
def forward():
    global playlist
    url = get_url(FORWARD)
    try:
        r = urllib2.urlopen(url)
        resp = utils.unserialize_response(r.read())
        if len(playlist) != 0:
            playlist.pop(0)
    except Exception:
        resp = {"success": False, "msg": "Error in Forwarding Song"}
    return resp
示例#20
0
 def check_song(self, song_hash):
     self._last_hb_ts = time.time() * MICROSECONDS
     content = utils.unserialize_response(request.get_data())
     command_epoch = content['command_epoch']
     if self._in_recovery:
         failover_resp = utils.format_rpc_response(False, CHECK, {}, \
                                              msg='Replica in recovery mode', \
                                              command_epoch=command_epoch)
         return utils.serialize_response(failover_resp)
     if song_hash in self._song_hashes:
         resp = utils.format_rpc_response(True, CHECK, \
                                          {'has_song': True, 'ip': self._ip}, \
                                          command_epoch = command_epoch)
     else:
         resp = utils.format_rpc_response(True, CHECK, {'ip': self._ip}, \
                                          command_epoch = command_epoch)
     return utils.serialize_response(resp)
 def load_song(self, song_hash):
     print "master client: load song"
     self.inc_client_req_id()
     command_info = {'command':LOAD, 'master_ip': self._ip, 'params':{'song_hash':song_hash}, 'client_req_id': self._client_req_id}
     if request.method == 'GET':
         if os.path.exists(utils.get_music_path(song_hash)):
             self._command_queue.put(command_info)
             return self.wait_on_master_music_service(LOAD)
         # song doesn't exist on master, get the song from the client
         else:
             return utils.serialize_response(utils.format_client_response(False, LOAD, {}, msg='Master does not have requested song'))
     elif request.method == 'POST':
         data = utils.unserialize_response(request.get_data())
         with open(utils.get_music_path(song_hash), 'w') as f:
             f.write(data['song_bytes'])
         self._command_queue.put(command_info)
         return self.wait_on_master_music_service(LOAD)
示例#22
0
def enqueue_song(song_path):
    load_song(song_path)
    assert (os.path.exists(song_path))
    with open(song_path, 'r') as f:
        song_bytes = f.read()
        song_hash = utils.hash_string(song_bytes)
    url = get_url(ENQUEUE) + "/" + song_hash
    try:
        r = urllib2.urlopen(url)
    except Exception:
        print "Error in Enqueue Song"
    else:
        master_response = utils.unserialize_response(r.read())
        if master_response['success'] == True:
            print song_path + ' has been enqueued'
        else:
            print song_path + ' cannot be enqueued'
        print master_response['client_req_id']
示例#23
0
def enqueue_song(song_path):
    load_song(song_path)
    assert(os.path.exists(song_path))
    with open(song_path, 'r') as f:
        song_bytes = f.read()
        song_hash = utils.hash_string(song_bytes)
    url = get_url(ENQUEUE) + "/" + song_hash
    try: 
        r = urllib2.urlopen(url)
    except Exception:
        print "Error in Enqueue Song"
    else:
        master_response = utils.unserialize_response(r.read())
        if master_response['success'] == True:
            print song_path + ' has been enqueued'
        else:
            print song_path + ' cannot be enqueued'
        print master_response['client_req_id']
示例#24
0
    def enqueue_song(self, song_hash):
        self._last_hb_ts = time.time() * MICROSECONDS
        content = utils.unserialize_response(request.get_data())
        command_epoch = content['command_epoch']
        master_post_hash = content['hashed_post_playlist']
        master_current_song = content['current_song']
        failover_resp = utils.format_rpc_response(False, ENQUEUE, {}, \
                                             msg='Replica in recovery mode', \
                                             command_epoch=command_epoch)
        if self._in_recovery:
            return utils.serialize_response(failover_resp)
        print "In Enqueue"
        print "Enqueue: " + str(self._current_song)

        replica_pre_hash = utils.hash_string(pickle.dumps(
            self._playlist_queue))
        if replica_pre_hash == master_post_hash and self._current_song == master_current_song:
            print "Already Performed Operation in Enqueue"
            repeat_resp = utils.format_rpc_response(False, ENQUEUE, {}, \
                                                 msg='Already performed operation', \
                                                 command_epoch=command_epoch)
            return utils.serialize_response(repeat_resp)

        # Do enqueue, check for failover mode
        song_not_exist = not os.path.exists(utils.get_music_path(song_hash))
        self._playlist_queue.append(song_hash)
        replica_post_hash = utils.hash_string(
            pickle.dumps(self._playlist_queue))
        inconsistent_queue = master_post_hash != replica_post_hash or \
                             master_current_song != self._current_song
        print "queue hashes match: " + str(
            master_post_hash == replica_post_hash)
        print "current song matches: " + str(
            master_current_song) + " == " + str(self._current_song)
        replica_failover = song_not_exist or inconsistent_queue
        if replica_failover:
            self._in_recovery = True
            return utils.serialize_response(failover_resp)
        master_time = content['time']
        self._master_timestamp = master_time
        resp = utils.format_rpc_response(True, ENQUEUE, {'enqueued': True}, \
                                         command_epoch=command_epoch)
        print str(resp)
        return utils.serialize_response(resp)
示例#25
0
    def dequeue_song(self):
        self._last_hb_ts = time.time() * MICROSECONDS
        content = utils.unserialize_response(request.get_data())
        command_epoch = content['command_epoch']
        master_post_hash = content['hashed_post_playlist']
        master_current_song = content['current_song']
        failover_resp = utils.format_rpc_response(False, DEQUEUE, {}, \
                                             msg='Replica in recovery mode', \
                                             command_epoch=command_epoch)
        if self._in_recovery:
            return utils.serialize_response(failover_resp)
        print "In Dequeue"

        replica_pre_hash = utils.hash_string(pickle.dumps(
            self._playlist_queue))
        if replica_pre_hash == master_post_hash and self._current_song == master_current_song and self._current_song != None:
            repeat_resp = utils.format_rpc_response(False, DEQUEUE, {}, \
                                                 msg='Already performed operation', \
                                                 command_epoch=command_epoch)
            return utils.serialize_response(repeat_resp)

        # Check for length 0 queue
        if len(self._playlist_queue) == 0:
            self._current_song = None
        else:
            self._current_song = self._playlist_queue.popleft()
        replica_post_hash = utils.hash_string(
            pickle.dumps(self._playlist_queue))

        if (replica_post_hash != master_post_hash
                or self._current_song != master_current_song):
            self._in_recovery = True
            return utils.serialize_response(failover_resp)
        master_time = content['time']
        self._master_timestamp = master_time
        resp = utils.format_rpc_response(True, DEQUEUE, {}, \
                                         msg='Successfully dequeued', \
                                         command_epoch=command_epoch)
        print "end of dequeue: current song for replica: " + str(
            self._current_song)
        return utils.serialize_response(resp)
示例#26
0
def enqueue(song_file):
    global playlist
    playlist.append(song_file)
    return 'songthing'
    song_path = MUSIC_DIR + song_file
    load_song(song_path)
    assert (os.path.exists(song_path))
    with open(song_path, 'r') as f:
        song_bytes = f.read()
        song_hash = utils.hash_string(song_bytes)
    url = get_url(ENQUEUE) + "/" + song_hash
    try:
        r = urllib2.urlopen(url)
    except Exception:
        print "Error in Enqueue Song"
    else:
        master_response = utils.unserialize_response(r.read())
        if master_response['success'] == True:
            print song_path + ' has been enqueued'
            playlist.append(song_file)
        else:
            print song_path + ' cannot be enqueued'
        print master_response['client_req_id']
    return 'enqueue'
示例#27
0
def enqueue(song_file):
    global playlist
    playlist.append(song_file)
    return 'songthing'
    song_path = MUSIC_DIR + song_file
    load_song(song_path)
    assert(os.path.exists(song_path))
    with open(song_path, 'r') as f:
        song_bytes = f.read()
        song_hash = utils.hash_string(song_bytes)
    url = get_url(ENQUEUE) + "/" + song_hash
    try: 
        r = urllib2.urlopen(url)
    except Exception:
        print "Error in Enqueue Song"
    else:
        master_response = utils.unserialize_response(r.read())
        if master_response['success'] == True:
            print song_path + ' has been enqueued'
            playlist.append(song_file)
        else:
            print song_path + ' cannot be enqueued'
        print master_response['client_req_id']
    return 'enqueue'
示例#28
0
 def load_song(self, song_hash):
     content = utils.unserialize_response(request.get_data())
     command_epoch = content['command_epoch']
     song_bytes = content['song_bytes']
     if self._in_recovery:
         failover_resp = utils.format_rpc_response(False, LOAD, {}, \
                                              msg='Replica in recovery mode', \
                                              command_epoch=command_epoch)
         self._last_hb_ts = time.time() * MICROSECONDS
         return utils.serialize_response(failover_resp)
     try:
         with open(utils.get_music_path(song_hash), 'w') as f:
             f.write(song_bytes)
     except Exception:
         resp = utils.format_rpc_response(False, LOAD, {}, \
                                          msg='Error occured in writing to replica', \
                                          command_epoch=command_epoch)
     else:
         self._song_hashes.add(song_hash)
         resp = utils.format_rpc_response(True, LOAD, \
                                          {'has_song': True, 'ip':self._ip}, \
                                          command_epoch=command_epoch)
     self._last_hb_ts = time.time() * MICROSECONDS
     return utils.serialize_response(resp)
 def reconnected_replica(self):
     print "RECONNECTING F*****G REPLICA"
     data = utils.unserialize_response(request.get_data())
     if data['msg'] == 'yo':
         return utils.serialize_response({'msg': 'yo'})
示例#30
0
 def reconnected_replica(self):
     print "RECONNECTING F*****G REPLICA"
     data = utils.unserialize_response(request.get_data())
     if data['msg'] == 'yo':
         return utils.serialize_response({'msg': 'yo'})
示例#31
0
  def run(self):
    self._data['master_ip'] = self._parent._master_ip

    request_data = utils.serialize_response(self._data)
    req = urllib2.Request(self._url, request_data) 
    start = int(round(time.time() * MICROSECONDS))
    # TODO: deal with timeout error
    response = urllib2.urlopen(req).read()
    end = int(round(time.time() * MICROSECONDS))

    try:
        response_data = utils.unserialize_response(response)
    except:
        # votes should fail to unserialize
        response_data = response
        if "yes" in response_data:
            self._parent.yes_votes[self._ip] = "yes"    
        return   
        
    # Short circuit out to prevent races, ignore timed out rpc calls
    print str(response_data)
    if 'command_epoch' in response_data and response_data['command_epoch'] != self._parent.command_epoch:
        return 
    response_command = response_data['command']
    response_success = response_data['success']
    response_params = response_data['params']
    assert(response_command == self._command)

    if response_success:
        # Response received from replica
        self._parent.rpc_response_acks += 1
        # Heartbeat time delay estimate
        if response_command == HB:
            # Replica_playing is True if playing or paused
            # False if song stopped
            replica_playing = int(response_params['replica_playing'])
            response_time = int(response_params['time'])
            self._parent.update_latency(self._ip, (end - start) / 2.0)
            if not replica_playing:
                self._parent.rpc_not_playing_acks += 1
            else:
                self._parent.rpc_playing_acks += 1
            # estimate for other computers clock time: their time, 
            # plus return network latency, which we approximate with (end-start)/2
            clock_estimate = response_time + (end - start) / 2
            diff = clock_estimate - end
            self._parent.update_clock_diff(self._ip, diff)
        # Check the offsets in PAUSE
        elif response_command == PAUSE:
            response_offset = int(response_params['offset'])
            if response_offset > -1:
                self._parent.rpc_offsets.append(response_offset)
        # Check if song has loaded/not loaded on replica
        elif response_command == LOAD:
            if 'has_song' in response_params:
                self._parent.rpc_loaded_ips.put(response_params['ip'] + ':' + REPLICA_PORT)
        elif response_command == CHECK:
            if 'has_song' in response_params:
                self._parent.rpc_loaded_ips.put(response_params['ip'] + ':' + REPLICA_PORT)
            else:
                self._parent.rpc_not_loaded_ips.put(response_params['ip'] + ':' + REPLICA_PORT)
    else:
        self._parent.rpc_not_playing_acks += 1
            
    if DEBUG:
        print "ip:" + self._ip + ":" + str(response_data)
示例#32
0
  def recover_state(self):
    status_update_dict = { "reset_election" : True }
    self._parent._state_queue.put(status_update_dict)
    num_master_retries = 10 + random.randint(0, 10)
    on_retry = 0
    while True:
      url = "http://" + self._parent._master_ip + ":" + REPLICA_FAIL_PORT + "/" + RECONNECT
      print "connecting to master failover service: " + url
      req = urllib2.Request(url, utils.serialize_response({'msg': 'yo'}))
      try:
        resp = urllib2.urlopen(req)
      except Exception:
        print "failed to reach master"
        on_retry += 1
        if on_retry > num_master_retries:
            return True # attempt to get elected again
        time.sleep(HEARTBEAT_PAUSE * 2)
      else:
        status_update_dict = { "new_master" : True }
        self._parent._state_queue.put(status_update_dict)
        response_data = utils.unserialize_response(resp.read())
        assert (response_data['msg'] == 'yo')
        break

    url = "http://" + self._parent._master_ip + ":" + REPLICA_FAIL_PORT + "/" + RECOVER
    data = {'song_hashes': []}

    # if music directory exists, get the filenames, keep extension on filenames!!!
    if os.path.exists(MUSIC_DIR): 
      for file_name in os.listdir(MUSIC_DIR):
        if (len(file_name) >= len(EXT)) and (file_name.count(EXT) != 0):
          data['song_hashes'].append(file_name)
    else:
      os.makedirs(MUSIC_DIR)

    req = urllib2.Request(url, utils.serialize_response(data))
    try:
      resp = urllib2.urlopen(req)
    except Exception:
      return False
    else:
      # TODO: add timestamp from master
      resp = resp.read()
      # response should be a ton of music mp3 files serialized into a dictionary
      response_data = utils.unserialize_response(resp)
      songs = response_data['params']['songs']
      master_queue = response_data['params']['master_queue']
      current_song = response_data['params']['current_song']
      self._parent._playlist_queue = master_queue
      self._parent._current_song = current_song
      #failed_file_names = []

      # need to update self._song_hashes in replica music server
      for file_name in songs:
        try:
            with open(file_name, 'w') as f:
              f.write(songs[file_name])
        except Exception:
            print 'song failed to download in replica failover'
            #failed_file_names.append(file_name)
        else:
            print 'successfully downloaded ' + file_name + ' in replica failover'
      return True
示例#33
0
    def play(self):
        self._last_hb_ts = time.time() * MICROSECONDS
        # Parse payload
        content = utils.unserialize_response(request.get_data())
        command_epoch = content['command_epoch']
        start_time_microsec = content['start_time']
        # Not sure when it would be -1 but double check
        assert (start_time_microsec >= 0)
        master_offset = 0
        if 'offset' in content:
            master_offset = int(content['offset'])
        # May be None, need to call .stop() then
        song_hash = content['song_hash']

        failover_mode_resp = utils.format_rpc_response(\
                            False, PLAY, {}, \
                            msg='Replica in recovery mode', \
                            command_epoch=command_epoch)

        # Case 1: Failover Mode already
        # Short circuit crucial to not starve failover_service process on
        # pygame_mixer object
        # Case 2: Song hash doesn't exist
        if self._in_recovery or (song_hash != None and not \
        os.path.exists(utils.get_music_path(song_hash))):
            self._in_recovery = True
            return utils.serialize_response(failover_mode_resp)

        # Could be in recovery mode at this point, wait for it to finish
        # before moving on (True)
        with self._pygame_lock:
            replica_offset = int(round(pygame.mixer.music.get_pos()))
            offset_diff = master_offset - replica_offset
            # Ideally if pygame.mixer.music.play(offset) works then these would not be
            # errors but alas such is life and we must go into recovery
            if master_offset > 0:
                # Case 3: Unpause command, but songs do not match
                # Case 4: Unpause command, but master's offset is way in future
                #         This should not happen since master determines
                #         offset as max of replica offsets
                if self._current_song != song_hash or \
                   abs(offset_diff) > MAX_OFFSET_DIF:
                    self._in_recovery = True
                    return utils.serialize_response(failover_mode_resp)
            elif song_hash != None:
                print "Loaded song"
                pygame.mixer.music.stop()
                pygame.mixer.music.load(utils.get_music_path(song_hash))

            # Adjust start_time_microsec to account for offset difference
            if master_offset > 0:
                start_time_microsec = \
                    start_time_microsec - (offset_diff*MILLISECONDS)

            # wait until start time, then play
            curr_replica_microsec = int(round(time.time() * MICROSECONDS))
            print song_hash
            print master_offset
            while (curr_replica_microsec + ALLOWED_REPLICA_BUFFER <
                   start_time_microsec):
                curr_replica_microsec = int(round(time.time() * MICROSECONDS))
            print "Song hash: " + str(song_hash)
            if song_hash == None:
                pygame.mixer.music.stop()
            elif master_offset == 0:
                pygame.mixer.music.play(1)
            else:
                pygame.mixer.music.unpause()
            time.sleep(2)  # necessary to allow mp3 thread to start?
            self._current_song = song_hash
            curr_replica_microsec = int(round(time.time() * MICROSECONDS))
            resp = utils.format_rpc_response(True, PLAY, \
                                             {'time': curr_replica_microsec }, \
                                             command_epoch='command_epoch')
            print "Play: " + str(self._current_song)
            return utils.serialize_response(resp)