def test_resume_producing_no_existing_manifest(self, node):
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
node.process_document = MagicMock(side_effect=EndOfData())
node.document_sequence.sequence_identifier = "testSeq"
fs_carriage.register(node)
fs_carriage.resume_producing()
assert node.process_document.called
def test_resume_producing_existing_manifest(self, node):
manifest_path = os.path.join(self.test_dir_path, "manifest_testSeq.txt")
with open(manifest_path, "w") as f:
f.write("00:00:00.123678,testSeq_177.xml")
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
node.process_document = MagicMock(side_effect=EndOfData())
node.document_sequence.sequence_identifier = "testSeq"
fs_carriage.register(node)
fs_carriage.resume_producing()
assert node.process_document.called
self.assertEqual(node.document_sequence.last_sequence_number, 177)
def test_emit_document(self):
data = 'test'
node = MagicMock(spec=IProducerNode)
node.provides.return_value = six.text_type
test_time = timedelta(hours=42,
minutes=42,
seconds=42,
milliseconds=67)
node.resume_producing.side_effect = EndOfData()
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
fs_carriage.register_producer_node(node)
fs_carriage.resume_producing()
fs_carriage.emit_data(data,
availability_time=test_time,
sequence_identifier='testSeq',
sequence_number=1,
time_base='clock',
clock_mode='local')
exported_document_path = os.path.join(self.test_dir_path,
'testSeq_1.xml')
assert os.path.exists(exported_document_path)
manifest_path = os.path.join(self.test_dir_path,
'manifest_testSeq.txt')
assert os.path.exists(manifest_path)
assert fs_carriage._default_clocks == {}
def given_buffer_delay(delay_offset, test_context, gen_document, temp_dir):
gen_document.availability_time = LimitedClockTimingType(
'00:00:00.0').timedelta
# the first delay node applies no delay
carriage_delay1 = FilesystemProducerImpl(os.path.join(temp_dir, 'initial'))
delay_float1 = LimitedClockTimingType(
'00:00:00.0').timedelta.total_seconds()
buffer_delay_node1 = BufferDelayNode(node_id='simple-delay-node',
producer_carriage=None,
fixed_delay=delay_float1)
ProducerNodeCarriageAdapter(producer_node=buffer_delay_node1,
producer_carriage=carriage_delay1)
buffer_delay_node1.process_document(
document=gen_document.get_xml(),
time_base=gen_document.time_base,
availability_time=gen_document.availability_time,
sequence_identifier=gen_document.sequence_identifier,
sequence_number=gen_document.sequence_number)
# the second delay node applies a delay of delay_offset
carriage_delay2 = FilesystemProducerImpl(os.path.join(temp_dir, 'buffer'))
delay_float2 = LimitedClockTimingType(
delay_offset).timedelta.total_seconds()
buffer_delay_node2 = BufferDelayNode(node_id='simple-delay-node',
producer_carriage=None,
fixed_delay=delay_float2)
ProducerNodeCarriageAdapter(producer_node=buffer_delay_node2,
producer_carriage=carriage_delay2)
buffer_delay_node2.process_document(
document=gen_document.get_xml(),
time_base=gen_document.time_base,
availability_time=gen_document.availability_time,
sequence_identifier=gen_document.sequence_identifier,
sequence_number=gen_document.sequence_number)
test_context['doc'] = gen_document
def test_resume_producing_no_existing_manifest(self, node):
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
node.process_document = MagicMock(side_effect=EndOfData())
node.document_sequence.sequence_identifier = "testSeq"
fs_carriage.register(node)
fs_carriage.resume_producing()
assert node.process_document.called
def test_emit_document(self):
document = MagicMock(sequence_identifier="testSeq", sequence_number=1)
document.get_xml = MagicMock(return_value="test")
node = MagicMock()
test_time = timedelta(hours=42, minutes=42, seconds=42, milliseconds=67)
node.reference_clock.get_time.return_value = test_time
node.process_document = MagicMock(side_effect=EndOfData())
node.document_sequence.sequence_identifier = "testSeq"
node.reference_clock.time_base = "clock"
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
fs_carriage.register(node)
fs_carriage.resume_producing()
fs_carriage.emit_document(document)
exported_document_path = os.path.join(self.test_dir_path, 'testSeq_1.xml')
assert os.path.exists(exported_document_path)
manifest_path = os.path.join(self.test_dir_path, 'manifest_testSeq.txt')
assert os.path.exists(manifest_path)
def test_msg_first_item_missing_availability(self):
data = 'live message without availability time'
node = MagicMock(spec=IProducerNode)
node.provides.return_value = six.text_type
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
fs_carriage.register_producer_node(node)
fs_carriage.emit_data(data, sequence_identifier='testSeq')
# Expected behaviour is to write the message file but fail to write the manifest.
assert os.listdir(self.test_dir_path) == ['testSeq_msg_1.xml']
assert fs_carriage._default_clocks == {}
def main():
args = parser.parse_args()
create_loggers()
do_export = False
if args.folder_export:
do_export = True
sub_consumer_impl = WebsocketConsumerCarriage()
sub_prod_impl = None
if do_export:
sub_prod_impl = FilesystemProducerImpl(args.folder_export)
else:
sub_prod_impl = WebsocketProducerCarriage()
reference_clock = LocalMachineClock()
reference_clock.clock_mode = 'local'
dist_node = DistributingNode(
node_id='distributing-node',
producer_carriage=sub_prod_impl,
consumer_carriage=sub_consumer_impl,
reference_clock=reference_clock
)
# This factory listens for incoming documents from the user input producer.
user_input_server_factory = UserInputServerFactory(
url=args.websocket_url,
consumer=TwistedConsumer(
custom_consumer=sub_consumer_impl
)
)
user_input_server_factory.protocol = UserInputServerProtocol
user_input_server_factory.listen()
if not do_export:
# This factory listens for any consumer to forward documents to.
broadcast_factory = BroadcastServerFactory("ws://127.0.0.1:9000")
broadcast_factory.protocol = BroadcastServerProtocol
broadcast_factory.listen()
TwistedPushProducer(
consumer=broadcast_factory,
custom_producer=sub_prod_impl
)
reactor.run()
def test_resume_producing_existing_manifest(self, node):
manifest_path = os.path.join(self.test_dir_path, "manifest_testSeq.txt")
with open(manifest_path, 'w') as f:
f.write("00:00:00.123678,testSeq_177.xml")
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
node.process_document = MagicMock(side_effect=EndOfData())
node.document_sequence.sequence_identifier = "testSeq"
fs_carriage.register(node)
fs_carriage.resume_producing()
assert node.process_document.called
self.assertEqual(node.document_sequence.last_sequence_number, 177)
def test_resume_producing_no_existing_manifest(self):
node = MagicMock(spec=IProducerNode)
node.provides.return_value = six.text_type
def side_effect():
raise EndOfData()
node.resume_producing.side_effect = side_effect
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
fs_carriage.register_producer_node(node)
fs_carriage.resume_producing()
node.resume_producing.assert_called_once()
def test_emit_document(self):
document = MagicMock(sequence_identifier="testSeq", sequence_number=1)
document.get_xml = MagicMock(return_value="test")
node = MagicMock()
test_time = timedelta(hours=42, minutes=42, seconds=42, milliseconds=67)
node.reference_clock.get_time.return_value = test_time
node.process_document = MagicMock(side_effect=EndOfData())
node.document_sequence.sequence_identifier = "testSeq"
node.reference_clock.time_base = "clock"
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
fs_carriage.register(node)
fs_carriage.resume_producing()
fs_carriage.emit_document(document)
exported_document_path = os.path.join(self.test_dir_path, "testSeq_1.xml")
assert os.path.exists(exported_document_path)
manifest_path = os.path.join(self.test_dir_path, "manifest_testSeq.txt")
assert os.path.exists(manifest_path)
def test_msg_mid_sequence_partial_missing_availability(self):
# This is a quirky edge case that should never really happen
# An acceptable workaround could be to take the the last received availability time - local clock
# offset for the sequence and compute an estimate using the local clock
#
# Since this was not a requirement the message gets discarded...
#
# There are also an unsettling number of odd defined/not defined combinations of arguments at this
# level, none of which is specified to trigger a particular response: such as receiving
# availability_time but no time_base...etc.
node = MagicMock(spec=IProducerNode)
node.provides.return_value = six.text_type
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
fs_carriage.register_producer_node(node)
test_time = timedelta(hours=42,
minutes=42,
seconds=42,
milliseconds=67)
data = 'document with availability'
fs_carriage.emit_data(data,
sequence_identifier='testSeq',
availability_time=test_time,
sequence_number=1,
time_base='clock',
clock_mode='local')
data = 'live message without availability time'
# This message does not have enough information to produce a reference clock by itself
fs_carriage.emit_data(data, sequence_identifier='testSeq')
assert len(os.listdir(
self.test_dir_path)) == 3 # document, message and manifest
exported_document_path = os.path.join(self.test_dir_path,
'testSeq_1.xml')
assert os.path.exists(exported_document_path)
exported_message_path = os.path.join(self.test_dir_path,
'testSeq_msg_1.xml')
assert os.path.exists(exported_message_path)
manifest_path = os.path.join(self.test_dir_path,
'manifest_testSeq.txt')
assert os.path.exists(manifest_path)
def test_msg_mid_sequence_missing_availability(self):
node = MagicMock(spec=IProducerNode)
node.provides.return_value = six.text_type
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
fs_carriage.register_producer_node(node)
data = 'document without availability'
fs_carriage.emit_data(data,
sequence_identifier='testSeq',
sequence_number=1,
time_base='clock',
clock_mode='local')
data = 'live message without availability time'
fs_carriage.emit_data(data, sequence_identifier='testSeq')
assert len(os.listdir(
self.test_dir_path)) == 3 # document, message and manifest
exported_document_path = os.path.join(self.test_dir_path,
'testSeq_1.xml')
assert os.path.exists(exported_document_path)
exported_message_path = os.path.join(self.test_dir_path,
'testSeq_msg_1.xml')
assert os.path.exists(exported_message_path)
manifest_path = os.path.join(self.test_dir_path,
'manifest_testSeq.txt')
assert os.path.exists(manifest_path)
def test_doc_missing_availability(self):
data = 'test document without availability time'
node = MagicMock(spec=IProducerNode)
node.provides.return_value = six.text_type
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
fs_carriage.register_producer_node(node)
fs_carriage.emit_data(data,
sequence_identifier='testSeq',
sequence_number=1,
time_base='clock',
clock_mode='local')
exported_document_path = os.path.join(self.test_dir_path,
'testSeq_1.xml')
assert os.path.exists(exported_document_path)
manifest_path = os.path.join(self.test_dir_path,
'manifest_testSeq.txt')
assert os.path.exists(manifest_path)
assert isinstance(fs_carriage._default_clocks['testSeq'], Clock)
def main():
create_loggers()
parsed_args = parser.parse_args()
do_export = False
if parsed_args.folder_export:
do_export = True
reference_clock = LocalMachineClock()
reference_clock.clock_mode = 'local'
document_sequence = EBUTT3DocumentSequence(
sequence_identifier='TestSequence1',
lang='en-GB',
reference_clock=reference_clock
)
if parsed_args.reference_clock:
subtitle_tokens = None # Instead of text we provide the availability time as content.
else:
# Let's read our example conversation
full_text = get_example_data('simple_producer.txt')
if do_export:
subtitle_tokens = iter(tokenize_english_document(full_text))
else:
# This makes the source cycle infinitely.
subtitle_tokens = cycle(tokenize_english_document(full_text))
# This object is used as flexible binding to the carriage mechanism and twisted integrated as dependency injection
prod_impl = None
if do_export:
prod_impl = FilesystemProducerImpl(parsed_args.folder_export)
else:
prod_impl = TwistedProducerImpl()
simple_producer = SimpleProducer(
node_id='simple-producer',
carriage_impl=prod_impl,
document_sequence=document_sequence,
input_blocks=subtitle_tokens
)
if do_export:
prod_impl.resume_producing()
else:
factory = wsFactory(u"ws://127.0.0.1:9000")
factory.protocol = StreamingServerProtocol
factory.listen()
# We are using a pull producer because it is the looping_task timer that triggers the production from the websocket
# level. Every time the factory gets a pull signal from the timer it tells the producer to generate data.
TwistedPullProducer(
consumer=factory,
custom_producer=prod_impl
)
looping_task = task.LoopingCall(factory.pull)
looping_task.start(2.0)
reactor.run()
def main():
create_loggers()
parsed_args = parser.parse_args()
sequence_identifier = 'TestSequence1'
do_export = False
if parsed_args.folder_export:
do_export = True
reference_clock = LocalMachineClock()
reference_clock.clock_mode = 'local'
document_sequence = EBUTT3DocumentSequence(
sequence_identifier=sequence_identifier,
lang='en-GB',
reference_clock=reference_clock
)
if parsed_args.reference_clock:
subtitle_tokens = None # Instead of text we provide the availability time as content.
else:
# Let's read our example conversation
full_text = get_example_data('simple_producer.txt')
if do_export:
subtitle_tokens = iter(tokenize_english_document(full_text))
else:
# This makes the source cycle infinitely.
subtitle_tokens = cycle(tokenize_english_document(full_text))
# This object is used as flexible binding to the carriage mechanism and twisted integrated as dependency injection
prod_impl = None
if do_export:
prod_impl = FilesystemProducerImpl(parsed_args.folder_export, reference_clock)
else:
prod_impl = WebsocketProducerCarriage()
prod_impl.sequence_identifier = sequence_identifier
simple_producer = SimpleProducer(
node_id='simple-producer',
producer_carriage=None,
document_sequence=document_sequence,
input_blocks=subtitle_tokens
)
# Chaining a converter
ProducerNodeCarriageAdapter(
producer_carriage=prod_impl,
producer_node=simple_producer
)
if do_export:
prod_impl.resume_producing()
else:
twisted_producer = TwistedWSPushProducer(
custom_producer=prod_impl
)
factory = BroadcastServerFactory(
url=u"ws://127.0.0.1:9000",
producer=twisted_producer
)
factory.protocol = BroadcastServerProtocol
factory.listen()
# Here we schedule in the simple producer to create content responding to a periodic interval timer.
looping_task = task.LoopingCall(simple_producer.process_document)
looping_task.start(2.0)
reactor.run()
def test_instantiation(self):
fs_carriage = FilesystemProducerImpl(self.test_dir_path)
self.assertIsInstance(fs_carriage, FilesystemProducerImpl)
