def set_block_payload(self, block_payload):
"""Sets the payload of the block to periodically add to the stream.
Args:
block_payload: A string, or None to clear the injected block.
"""
self._lock.acquire()
try:
if block_payload is None:
self._message_to_inject = None
# TODO(trow): Can we inject something into the stream to
# clear the previously streamed tags?
return
# If the same payload gets set twice, do nothing
if (self._message_to_inject is not None
and self._message_to_inject.payload == block_payload):
return
msg = message.Message()
msg.message_type = message.BLOCK
msg.payload = block_payload
self._message_to_inject = msg
# Reset the countdown; this ensures that the new block will
# be injected immediately.
self._countdown_ms = 0
finally:
self._lock.release()
def data_to_messages(self, data, block_size=137):
message_list = []
offset = 0
while data:
msg = message.Message()
msg.payload, data = data[:block_size], data[block_size:]
msg.message_type = message.BLOCK
msg.connection_id = 123
msg.connection_offset = offset
msg.start_timestamp_ms = 100000 # Dummy start time
msg.end_timestamp_ms = 100050 # Dummy end time
offset += len(msg.payload)
message_list.append(msg)
return message_list
def test_rollover(self):
# Set up a test archiver.
src = message.MessageSource()
arch = archiver.Archiver(src)
arch.ROOT_DIR = "/tmp/archiver_test/%d" % time.time()
arch.loop_in_thread()
# The archiver's rollover count should start out at zero.
self.assertEqual(0, arch.rollover_count)
# Construct a test frame.
test_ms = 1262648655000 # Jan 4, 2010, 5:43pm
msg = message.Message()
msg.message_type = message.FRAME
msg.payload = mp3_frame.dead_air(1) # 1ms = 1 frame
msg.mp3_header = mp3_header.parse(msg.payload)
msg.start_timestamp_ms = test_ms
msg.end_timestamp_ms = test_ms + msg.mp3_header.duration_ms
# Add the message to our source, twice, and wait for it to be
# processed. We add the message twice to work around the bug
# whereby mutagen will choke on a single-frame MP3.
src._add_message(msg)
src._add_message(msg)
src.wait_until_empty()
# The archiver's rollover count should still be zero.
self.assertEqual(0, arch.rollover_count)
# Now advance the timestamp by one hour and re-add the message.
# That should trigger a rollover.
one_hour_in_ms = 1000 * 60 * 60
msg.start_timestamp_ms += one_hour_in_ms
msg.end_timestamp_ms += one_hour_in_ms
src._add_message(msg)
src.wait_until_empty()
# Now there should have been a rollover.
self.assertEqual(1, arch.rollover_count)
src._add_message(msg) # Again, to avoid the mutagen bug.
# Shut down the archiver's thread, then wait for it to terminate.
src._add_stop_message()
arch.wait()
def _process_message(self, msg):
if msg.message_type != message.BLOCK:
# Clear the buffer and reset our various time-tracking
# variables before passing the message through.
self._buffered = []
self._start_timestamp_ms = None
self._elapsed_frame_ms = 0
self._add_message(msg)
return
# If necessary, remember the start time of this message.
if self._start_timestamp_ms is None:
self._start_timestamp_ms = msg.start_timestamp_ms
# Take the buffered data and split it into MP3 frames.
self._buffered.append(msg.payload)
frames = list(
mp3_frame.split_blocks(iter(self._buffered),
expected_hdr=self._expected_hdr))
self._buffered = []
# If any of the last frames appear to be junk, they might
# contain a truncated frame. If so, stick them back onto
# the buffer.
while frames:
last_hdr, last_buffer = frames[-1]
if last_hdr is not None:
break
self._buffered.insert(0, last_buffer)
frames.pop()
# Turn the frames into FRAME messages and add them to our
# queue.
start_ms = (self._start_timestamp_ms + self._start_adjustment_ms +
self._elapsed_frame_ms)
for hdr, data in frames:
new_msg = message.Message()
if hdr is None:
new_msg.message_type = message.BLOCK
duration_ms = 0
else:
new_msg.message_type = message.FRAME
duration_ms = hdr.duration_ms
self._elapsed_frame_ms += duration_ms
new_msg.connection_id = msg.connection_id
new_msg.connection_offset = msg.connection_offset
new_msg.mp3_header = hdr
new_msg.payload = data
# Now set the start and end timestamps; these are our best
# approximations, and might do alarming things like jump
# backward in time.
new_msg.start_timestamp_ms = int(start_ms)
new_msg.end_timestamp_ms = int(start_ms + duration_ms)
start_ms += duration_ms
self._add_message(new_msg)
# Now let's see if we need to modify our start-time adjustment.
# First, compute the total wall-clock time so far.
wall_clock_ms = msg.end_timestamp_ms - self._start_timestamp_ms
# Now find the difference between the elaped wall-clock time
# and the elapsed time implied by the frames.
delta_ms = wall_clock_ms - self._elapsed_frame_ms
# If the observed delta is negative (indicating that we've
# seen more frames than would normally be possible given how
# much time has passed), our new fudge factor will be the
# average of the observed discepancy and the previous fudge
# factor.
if delta_ms < 0:
self._start_adjustment_ms = (self._start_adjustment_ms +
delta_ms) / 2
def test_basic_injection(self):
# A mock MP3 frame.
mock_frame = message.Message()
mock_frame.message_type = message.FRAME
mock_frame.payload = "mock frame"
mock_frame.mp3_header = mp3_header.MP3Header()
mock_frame.mp3_header.duration_ms = 14000 # A 14-second-long frame
# A mock block.
mock_block = message.Message()
mock_block.message_type = message.BLOCK
mock_block.payload = "mock block"
src = message.MessageSource()
inj = block_injector.BlockInjector(src)
inj.loop_in_thread()
# Add 3 frames; since we haven't set a block payload yet, the
# injector should just let these pass through
for _ in xrange(3):
src._add_message(mock_frame)
src.wait_until_empty()
# Define a block to be injected.
injected_one = "Injected Payload"
inj.set_block_payload(injected_one)
# Now add 5 more frames. A block should be injected after the first
# (i.e. immediately) and fourth (i.e. after >30s elapse) frames.
for _ in xrange(5):
src._add_message(mock_frame)
src._add_message(mock_block)
# Also add our mock block. Blocks should pass through the
# injector without having any effect.
src.wait_until_empty()
# Define another block to be injected.
injected_two = "Another injected payload"
inj.set_block_payload(injected_two)
# Now add 5 more frames. Once again, a block should be
# injected after the first and fourth frames.
for _ in xrange(5):
src._add_message(mock_frame)
src.wait_until_empty()
# Clear out the injected block.
inj.set_block_payload(None)
# Now add 10 more frames. Since we cleared our injected block
# payload, nothing should be injected.
for _ in xrange(10):
src._add_message(mock_frame)
# Stop the source, and wait for the injector to settle.
src._add_stop_message()
inj.wait()
# Now pull all of the messages out of our injector, and check that
# we got what we expected.
all_messages = list(inj.get_all_messages())
# Our initial three frames, plus one more.
for _ in xrange(4):
self.assertTrue(all_messages.pop(0) is mock_frame)
# Our first injected block.
msg = all_messages.pop(0)
self.assertEqual(message.BLOCK, msg.message_type)
self.assertEqual(injected_one, msg.payload)
# Three more frames.
for _ in xrange(3):
self.assertTrue(all_messages.pop(0) is mock_frame)
# The next injected block.
msg = all_messages.pop(0)
self.assertEqual(message.BLOCK, msg.message_type)
self.assertEqual(injected_one, msg.payload)
# Another frame.
self.assertTrue(all_messages.pop(0) is mock_frame)
# Our mock block
self.assertTrue(all_messages.pop(0) is mock_block)
# Another frame.
self.assertTrue(all_messages.pop(0) is mock_frame)
# Another injected block, this time with our second payload.
msg = all_messages.pop(0)
self.assertEqual(message.BLOCK, msg.message_type)
self.assertEqual(injected_two, msg.payload)
# Three more frames.
for _ in xrange(3):
self.assertTrue(all_messages.pop(0) is mock_frame)
# The next injected block.
msg = all_messages.pop(0)
self.assertEqual(message.BLOCK, msg.message_type)
self.assertEqual(injected_two, msg.payload)
# Eleven more frames.
for _ in xrange(11):
self.assertTrue(all_messages.pop(0) is mock_frame)
# Finally, our end-of-stream marker.
msg = all_messages.pop(0)
self.assertTrue(msg.is_end_of_stream())
# That's all folks!
self.assertEqual(0, len(all_messages))
def _start_message(self):
"""Returns a new message."""
msg = message.Message()
msg.start_timestamp_ms = self._now_ms()
return msg
def test_basic(self):
# Create a test MP3Writer.
test_prefix = "/tmp/mp3_writer_test"
test_start_ms = 1262479500000
writer = mp3_writer.MP3Writer(prefix=test_prefix,
start_ms=test_start_ms)
self.assertEqual("/tmp/mp3_writer_test-20100102-184500.000.mp3",
writer.path)
# Set up a dummy MP3 frame (corresonding to our dead air)
# message and repeatedly add it to the writer.
msg = message.Message()
msg.message_type = message.FRAME
msg.payload = mp3_frame.dead_air(1) # 1 ms => a single frame
msg.mp3_header = mp3_header.parse(msg.payload)
# Set up a dummy block message.
block_msg = message.Message()
block_msg.message_type = message.BLOCK
block_msg.payload = "FOO BAR BAZ"
# Now repeatedly add the test message to the writer.
time_ms = test_start_ms
for i in xrange(10):
msg.start_timestamp_ms = time_ms
time_ms += msg.mp3_header.duration_ms
msg.end_timestamp_ms = time_ms
writer.write(msg)
# Also add the block message in. It should be ignored.
writer.write(block_msg)
# At this point the duration should be 10x the individual frame
# duration.
self.assertAlmostEqual(10*msg.mp3_header.duration_ms,
writer.duration_ms)
# We should also have the correct frame count and size.
self.assertEqual(10, writer.frame_count)
self.assertEqual(10*len(msg.payload), writer.frame_size)
# The file should be fully flushed out to disk, so we should be
# able to inspect it with mutagen and see something with the
# expected duration and tags.
partial_mp3 = mutagen.mp3.MP3(writer.path)
self.assertAlmostEqual(writer.duration_ms/1000, # mutagen uses seconds
partial_mp3.info.length)
# These are only some of the tags.
self.assertTrue("TRSN" in partial_mp3)
self.assertEqual([u"CHIRP Radio"], partial_mp3["TRSN"].text)
self.assertTrue("TIT1" in partial_mp3)
self.assertEqual([u"20100102-184500.000 to ???????????????????"],
partial_mp3["TIT1"].text)
# Now close the writer. After that, we should be able to open
# it with mutagen and see the final tags.
writer.close()
final_mp3 = mutagen.mp3.MP3(writer.path)
self.assertAlmostEqual(writer.duration_ms/1000, final_mp3.info.length)
# Check finalized title.
self.assertEqual([u"20100102-184500.000 to 20100102-184500.261"],
final_mp3["TIT1"].text)
# Check frame count.
self.assertTrue(constants.TXXX_FRAME_COUNT_KEY in final_mp3)
self.assertEqual([unicode(writer.frame_count)],
final_mp3[constants.TXXX_FRAME_COUNT_KEY].text)
# Check frame size.
self.assertTrue(constants.TXXX_FRAME_SIZE_KEY in final_mp3)
self.assertEqual([unicode(writer.frame_size)],
final_mp3[constants.TXXX_FRAME_SIZE_KEY].text)
# Check UFID.
self.assertTrue(constants.MUTAGEN_UFID_KEY in final_mp3)
self.assertEqual(
"volff/20100102-184500/8f5bb4f4b0ded8d29baa778f121ef3063db7a3f7",
final_mp3[constants.MUTAGEN_UFID_KEY].data)
# It should be safe to call close twice.
writer.close()