def test_consume_from_balancer_should_transcode_to_audio(tmpdir):
server = Server(M3U8_HOST, M3U8_PORT)
playlist = 'real'
uri = '/real_content.m3u8'
playlists = {'streams': {playlist: {'input-path': uri, 'servers': [server]}},
'actions': [{'type': 'transcode',
'input': playlist,
'output': {'audio': {
"transcode": {
"path": "transcode.m3u8",
"audio-bitrate": "64000",
"bandwidth": "65000"
}
}}}]}
b = Balancer()
b.update(get_servers(playlists))
hlsclient.consumer.consume_from_balancer(b, playlists, str(tmpdir))
expected_created = ['real_content.m3u8', 'sample.ts', 'transcode.m3u8', 'sample.aac']
resources_created = os.listdir(str(tmpdir))
assert sorted(expected_created) == sorted(resources_created)
original_m3u8 = tmpdir.join('real_content.m3u8').read()
expected_audio_m3u8 = original_m3u8.replace('.ts', '.aac')
assert expected_audio_m3u8 == tmpdir.join('transcode.m3u8').read()
def test_paths_can_be_removed():
PATH = '/path'
SERVERS = ['http://server1', 'http://server2', 'http://server3']
paths = {PATH: SERVERS}
b = Balancer()
b.update(paths)
b.update({})
assert [] == list(b.actives)
def test_balancer_returns_active_server_if_its_the_only_one():
PATH = '/path'
SERVER = FMS('http://server', port=80)
paths = {PATH: [SERVER]}
b = Balancer()
b.update(paths)
active_playlists = list(b.actives)
assert 1 == len(active_playlists)
assert PATH == active_playlists[0].key
assert SERVER == active_playlists[0].server
def test_balancer_returns_active_server_if_its_the_only_one():
PATH = '/path'
SERVER = FMS('http://server', port=80)
paths = {PATH: [SERVER]}
b = Balancer()
b.update(paths)
active_playlists = list(b.actives)
assert 1 == len(active_playlists)
assert PATH == active_playlists[0].key
assert SERVER == active_playlists[0].server
def test_balancer_supports_multiple_paths():
PATH1 = '/path1'
PATH2 = '/path2'
SERVER = 'http://server'
paths = {PATH1: [SERVER], PATH2: [SERVER]}
b = Balancer()
b.update(paths)
paths = sorted(s.key for s in b.actives)
assert 2 == len(paths)
assert PATH1 == paths[0]
assert PATH2 == paths[1]
def test_balancer_supports_multiple_paths():
PATH1 = '/path1'
PATH2 = '/path2'
SERVER = 'http://server'
paths = {PATH1: [SERVER], PATH2: [SERVER]}
b = Balancer()
b.update(paths)
paths = sorted(s.key for s in b.actives)
assert 2 == len(paths)
assert PATH1 == paths[0]
assert PATH2 == paths[1]
def setup(self):
helpers.setup_logging(self.config, "worker for {}".format(self.playlist))
logging.debug('HLS CLIENT Started for {}'.format(self.playlist))
self.destination = self.config.get('hlsclient', 'destination')
self.encrypt = self.config.getboolean('hlsclient', 'encrypt')
not_modified_tolerance = self.config.getint('hlsclient', 'not_modified_tolerance')
self.balancer = Balancer(not_modified_tolerance)
ttl = datetime.timedelta(seconds=random.randint(1, MAX_TTL_IN_SECONDS))
self.death_time = datetime.datetime.now() + ttl
def test_notify_error_should_rotate_servers_while_there_are_available_servers():
PATH1 = '/path1'
PATH2 = '/path2'
SERVER1 = 'http://server1'
SERVER2 = 'http://server2'
SERVERS = [SERVER1, SERVER2]
paths = {PATH1: SERVERS, PATH2: SERVERS}
b = Balancer()
b.update(paths)
b.notify_error()
b.notify_error()
assert list(b.actives) == [PlaylistResource(SERVER1, PATH1), PlaylistResource(SERVER1, PATH2)]
def test_consume_from_balancer_should_not_report_content_modified_if_there_are_no_changes(tmpdir):
server = Server(M3U8_HOST, M3U8_PORT)
playlist = 'low'
uri = '/low.m3u8'
playlists = {'streams': {playlist: {'input-path': uri, 'servers': [server]}}}
b = Balancer()
b.update(get_servers(playlists))
hlsclient.consumer.consume_from_balancer(b, playlists, str(tmpdir))
modified = []
b.notify_modified = lambda: modified.append("MODIFIED")
hlsclient.consumer.consume_from_balancer(b, playlists, str(tmpdir))
assert modified == []
def test_consume_from_balancer_should_timeout(tmpdir, monkeypatch):
server = Server(M3U8_HOST, M3U8_PORT)
playlist = 'slow'
uri = '/slow.m3u8'
playlists = {'streams': {playlist: {'input-path': uri, 'servers': [server]}}}
errors = []
b = Balancer()
b.update(get_servers(playlists))
b.notify_error = lambda: errors.append("ERROR")
monkeypatch.setattr(logging, 'warning', lambda warn: 0) # just to hide hlsclient warning
hlsclient.consumer.consume_from_balancer(b, playlists, str(tmpdir))
assert errors == ["ERROR"]
def test_if_server_fails_for_any_stream_all_streams_should_switch_server():
PATH1 = '/path1'
PATH2 = '/path2'
SERVER1 = 'http://server1'
SERVER2 = 'http://server2'
SERVERS = [SERVER1, SERVER2]
paths = {PATH1: SERVERS, PATH2: SERVERS}
b = Balancer()
b.update(paths)
assert list(b.actives) == [PlaylistResource(SERVER1, PATH1), PlaylistResource(SERVER1, PATH2)]
b.notify_error()
assert list(b.actives) == [PlaylistResource(SERVER2, PATH1), PlaylistResource(SERVER2, PATH2)]
def test_consume_from_balancer_should_report_content_modified(tmpdir):
server = Server(M3U8_HOST, M3U8_PORT)
playlist = 'low'
uri = '/low.m3u8'
playlists = {'streams': {playlist: {'input-path': uri, 'servers': [server]}}}
modified = []
b = Balancer()
b.update(get_servers(playlists))
b.notify_modified = lambda: modified.append("MODIFIED")
hlsclient.consumer.consume_from_balancer(b, playlists, str(tmpdir))
assert modified == ["MODIFIED"]
expected_created = ['low.m3u8', 'low1.ts', 'low2.ts']
resources_created = os.listdir(str(tmpdir))
assert sorted(expected_created) == sorted(resources_created)
for filename in resources_created:
assert stat.S_IMODE(os.stat(str(tmpdir.join(filename))).st_mode) == 0644
def test_active_server_does_not_change_if_paths_updated():
PATH = '/path'
SERVERS = ['http://server1', 'http://server2', 'http://server3']
paths = {PATH: SERVERS}
b = Balancer()
b.update(paths)
# Notify that active server has failed
b.notify_error()
assert [SERVERS[1]] == [s.server for s in b.actives]
b.update(paths)
assert [SERVERS[1]] == [s.server for s in b.actives]
def test_active_server_changes_if_error_detected():
PATH = '/path'
SERVERS = ['http://server1', 'http://server2', 'http://server3']
paths = {PATH: SERVERS}
b = Balancer()
b.update(paths)
# Notify that the active server has failed
assert [SERVERS[0]] == [s.server for s in b.actives]
b.notify_error()
# Assert that the backups assume
assert [SERVERS[1]] == [s.server for s in b.actives]
b.notify_error()
assert [SERVERS[2]] == [s.server for s in b.actives]
# Assert that the first server resumes if backup fails
b.notify_error()
assert [SERVERS[0]] == [s.server for s in b.actives]
def test_paths_can_be_removed():
PATH = '/path'
SERVERS = ['http://server1', 'http://server2', 'http://server3']
paths = {PATH: SERVERS}
b = Balancer()
b.update(paths)
b.update({})
assert [] == list(b.actives)
def test_active_server_does_not_change_if_paths_updated():
PATH = '/path'
SERVERS = ['http://server1', 'http://server2', 'http://server3']
paths = {PATH: SERVERS}
b = Balancer()
b.update(paths)
# Notify that active server has failed
b.notify_error()
assert [SERVERS[1]] == [s.server for s in b.actives]
b.update(paths)
assert [SERVERS[1]] == [s.server for s in b.actives]
def test_active_server_changes_if_error_detected():
PATH = '/path'
SERVERS = ['http://server1', 'http://server2', 'http://server3']
paths = {PATH: SERVERS}
b = Balancer()
b.update(paths)
# Notify that the active server has failed
assert [SERVERS[0]] == [s.server for s in b.actives]
b.notify_error()
# Assert that the backups assume
assert [SERVERS[1]] == [s.server for s in b.actives]
b.notify_error()
assert [SERVERS[2]] == [s.server for s in b.actives]
# Assert that the first server resumes if backup fails
b.notify_error()
assert [SERVERS[0]] == [s.server for s in b.actives]
def test_active_server_changes_if_playlist_not_modified_for_a_while(
monkeypatch):
PATH = '/path'
SERVERS = ['http://server1', 'http://server2']
paths = {PATH: SERVERS}
b = Balancer()
b.update(paths)
now = datetime.datetime.now()
assert [SERVERS[0]] == [s.server for s in b.actives]
b.notify_modified()
# 20 seconds later and playlist has not changed
monkeypatch.setattr(b, '_now',
lambda: now + datetime.timedelta(seconds=20))
assert [SERVERS[1]] == [s.server for s in b.actives]
# more 20 seconds later but backup is being updated
monkeypatch.setattr(b, '_now',
lambda: now + datetime.timedelta(seconds=40))
b.notify_modified()
assert [SERVERS[1]] == [s.server for s in b.actives]
def test_active_server_changes_if_playlist_not_modified_for_a_while(monkeypatch):
PATH = '/path'
SERVERS = ['http://server1', 'http://server2']
paths = {PATH: SERVERS}
b = Balancer()
b.update(paths)
now = datetime.datetime.now()
assert [SERVERS[0]] == [s.server for s in b.actives]
b.notify_modified()
# 20 seconds later and playlist has not changed
monkeypatch.setattr(b, '_now', lambda: now + datetime.timedelta(seconds=20))
assert [SERVERS[1]] == [s.server for s in b.actives]
# more 20 seconds later but backup is being updated
monkeypatch.setattr(b, '_now', lambda: now + datetime.timedelta(seconds=40))
b.notify_modified()
assert [SERVERS[1]] == [s.server for s in b.actives]
def test_notify_error_should_rotate_servers_while_there_are_available_servers(
):
PATH1 = '/path1'
PATH2 = '/path2'
SERVER1 = 'http://server1'
SERVER2 = 'http://server2'
SERVERS = [SERVER1, SERVER2]
paths = {PATH1: SERVERS, PATH2: SERVERS}
b = Balancer()
b.update(paths)
b.notify_error()
b.notify_error()
assert list(b.actives) == [
PlaylistResource(SERVER1, PATH1),
PlaylistResource(SERVER1, PATH2)
]
def test_if_server_fails_for_any_stream_all_streams_should_switch_server():
PATH1 = '/path1'
PATH2 = '/path2'
SERVER1 = 'http://server1'
SERVER2 = 'http://server2'
SERVERS = [SERVER1, SERVER2]
paths = {PATH1: SERVERS, PATH2: SERVERS}
b = Balancer()
b.update(paths)
assert list(b.actives) == [
PlaylistResource(SERVER1, PATH1),
PlaylistResource(SERVER1, PATH2)
]
b.notify_error()
assert list(b.actives) == [
PlaylistResource(SERVER2, PATH1),
PlaylistResource(SERVER2, PATH2)
]
class PlaylistWorker(object):
def __init__(self, playlist, is_variant=False):
self.playlist = playlist
self.is_variant = is_variant
self.config = helpers.load_config()
self.setup_lock()
def setup(self):
helpers.setup_logging(self.config, "worker for {}".format(self.playlist))
logging.debug('HLS CLIENT Started for {}'.format(self.playlist))
self.destination = self.config.get('hlsclient', 'destination')
self.encrypt = self.config.getboolean('hlsclient', 'encrypt')
not_modified_tolerance = self.config.getint('hlsclient', 'not_modified_tolerance')
self.balancer = Balancer(not_modified_tolerance)
ttl = datetime.timedelta(seconds=random.randint(1, MAX_TTL_IN_SECONDS))
self.death_time = datetime.datetime.now() + ttl
def run_forever(self):
self.setup()
signal.signal(signal.SIGTERM, self.interrupted)
while self.should_run():
try:
self.run_if_locking()
time.sleep(0.1)
except LockTimeout:
logging.debug("Unable to acquire lock")
except KeyboardInterrupt:
logging.debug('Quitting...')
break
except Exception:
logging.exception('An unknown error happened')
self.stop()
def run(self):
playlists = discover_playlists(self.config)
worker_playlists = self.filter_playlists_for_worker(playlists)
if not worker_playlists['streams']:
logging.warning("Playlist is not available anymore")
self.stop()
paths = get_servers(worker_playlists)
self.balancer.update(paths)
consume_from_balancer(self.balancer,
worker_playlists,
self.destination,
self.encrypt)
def filter_playlists_for_worker(self, playlists):
if self.is_variant:
combine_actions = get_actions(playlists, "combine")
my_combine_actions = [action for action in combine_actions if action['output'] == self.playlist]
my_inputs = [action['input'] for action in my_combine_actions]
streams = itertools.chain(*my_inputs)
streams = [s for s in streams if s in playlists['streams']] # transcoded playlists are ignored
elif self.playlist in playlists['streams']:
streams = [self.playlist]
else:
streams = []
result = playlists.copy()
result["streams"] = {stream: playlists['streams'][stream] for stream in streams}
return result
def should_run(self):
should_live = datetime.datetime.now() < self.death_time
if not should_live:
logging.info("Worker {} should die now!".format(self.worker_id()))
return should_live
def interrupted(self, *args):
logging.info('Interrupted. Releasing lock.')
self.stop()
def setup_lock(self):
lock_path = self.lock_path()
self.lock_timeout = self.config.getint('lock', 'timeout')
self.lock_expiration = self.config.getint('lock', 'expiration')
self.lock = ExpiringLinkLockFile(lock_path)
def lock_path(self):
return '{0}.{1}'.format(self.config.get('lock', 'path'), self.worker_id())
def worker_id(self):
return hashlib.md5(self.playlist).hexdigest()
def run_if_locking(self):
if self.other_is_running():
logging.warning("Someone else acquired the lock")
self.stop()
return
if not self.lock.is_locked():
self.lock.acquire(timeout=self.lock_timeout)
if self.lock.i_am_locking():
self.lock.update_lock()
self.run()
def other_is_running(self):
other = self.lock.is_locked() and not self.lock.i_am_locking()
if other and self.lock.expired(tolerance=self.lock_expiration):
logging.warning("Lock expired. Breaking it")
self.lock.break_lock()
return False
return other
def stop(self):
try:
self.lock.release_if_locking()
finally:
sys.exit(0)
