def test_extract_many_attributes_split_words(self):
data_in = """
TIME CHIP SERIAL IS: 5ABF3. OTHER VALUE IS: 312. SOME TRAILING TEXT
"""
extractor = ppln.ExtractManyAttributesSplitWords(\
factory=self.factory, split_on_str=". ", attr_delim='col')
extractor_filter = extractor.getf('attribute_extractor')
sink = df.Sink()
extractor_filter.next_filter = sink
extractor.send(dfb.DataPacket(data_in))
#TEST PACKET NOT FILTER
self.assertEqual(sink.results[0].time_chip_serial_is, '5ABF3')
self.assertEqual(sink.results[1].other_value_is, '312')
def setUp(self):
self.factory = ff.DemoFilterFactory()
self.packet1 = dfb.DataPacket(height=1)
self.packet2 = dfb.DataPacket(height=2)
self.packet3 = dfb.DataPacket(height=3)
self.packet4 = dfb.DataPacket(height=4)
self.packet5 = dfb.DataPacket(height=5)
self.packet6 = dfb.DataPacket(height=9)
def test_01(self):
##inner_bar = ppln_demo.InnerBar(factory=self.factory,
##temperature=15,
##speed=95)
small_pipe_baz = ppln_demo.SmallPipeBaz(factory=self.factory,
temperature=15,
speed=95)
print '\n**14420**', 60 * '-'
print '\n'.join(small_pipe_baz.module_loc._python_code_lines)
print '**14420**', 60 * '-'
packet1 = dfb.DataPacket('hello')
small_pipe_baz.send(packet1)
new_temp = 34
print '\n**14460** Changing temperature from %d to %d' % (
small_pipe_baz.temperature, new_temp)
small_pipe_baz.temperature = new_temp
packet2 = dfb.DataPacket('and')
small_pipe_baz.send(packet2)
new_speed = 700
print '\n**14470** Changing speed from %d to %d' % (
small_pipe_baz.speed, new_speed)
small_pipe_baz.speed = new_speed
packet3 = dfb.DataPacket('goodbye')
small_pipe_baz.send(packet3)
new_SPEED = 20
print '\n**14495** Changing SPEED from %s to %s' % (
small_pipe_baz.emb_module.SPEED, new_SPEED)
small_pipe_baz.emb_module.SPEED = new_SPEED
packet4 = dfb.DataPacket('world')
small_pipe_baz.send(packet4)
small_pipe_baz.shut_down()
def test_dynamic_pipeline2(self):
# Set dynamic=True by calling make_dynamic()
pipeline23 = ppln.DynamicPipeline(factory=self.factory,
config=self.config1,
foo='joe')
self.assertEquals(pipeline23.foo, 'joe')
self.assertEquals(pipeline23.bar, 27)
pipeline23.bar = '%SPEED'
# All u/c params should default to dfb.k_unset before packet
# is sent through
self.assertEquals(pipeline23.bar, dfb.k_unset)
# Send packet to activate py_set_param
packet = dfb.DataPacket('goodbye')
pipeline23.send(packet)
self.assertEquals(pipeline23.bar, 170)
def test_dynamic_pipeline1(self):
# Set dynamic=True in the kwargs passed in
pipeline22 = ppln.DynamicPipeline(factory=self.factory,
config=self.config1,
foo='jim')
self.assertEquals(pipeline22.foo, 'jim')
self.assertEquals(pipeline22.bar, 27)
pipeline22.bar = '%SPEED'
# All u/c params should default to dfb.k_unset before packet
# is sent through
self.assertEquals(pipeline22.bar, dfb.k_unset)
# Send packet to activate py_set_param
packet = dfb.DataPacket('hello')
pipeline22.send(packet)
self.assertEquals(pipeline22.bar, 170)
def setUp(self):
self.factory = ff.DemoFilterFactory()
self.pipeline1 = ppln.Pipeline(factory=self.factory,
config=self.config)
self.packet1 = dfb.DataPacket(height=1)
self.packet2 = dfb.DataPacket(height=2)
self.packet3 = dfb.DataPacket(height=3)
self.packet4 = dfb.DataPacket(height=4)
self.packet5 = dfb.DataPacket(height=5)
self.packet6 = dfb.DataPacket(height=9)
def test_dynamic_filter_in_static_pipeline3(self):
# Batch is set to be reversibly dynamic in config
config3 = '''\
[--main--]
ftype = demo_static_pipeline3
description = Static pipeline with reversible dynamic filter
keys = foo, bar:43
#dynamic = true
[py_set_param]
print '**16130** Setting BATCH_SIZE to 256'
BATCH_SIZE_SOMETHING = 256
BATCH_SIZE_OTHER = 128
[distill_something]
ftype = distill_header
dynamic = true
[distill_other]
ftype = distill_header
dynamic = true
[--route--]
py_set_param >>>
distill_something:%BATCH_SIZE_SOMETHING >>>
(pass_through)
distill_other:%BATCH_SIZE_OTHER >>>
(pass_through)
sink
'''
pipeline25 = ppln.Pipeline(factory=self.factory,
config=config3,
foo='jim')
self.assertEquals(pipeline25.foo, 'jim')
self.assertEquals(pipeline25.bar, 43)
something_filter = pipeline25.getf('distill_something')
other_filter = pipeline25.getf('distill_other')
# Dynamic value not yet set
self.assertEquals(something_filter.header_size, dfb.k_unset)
# Send packet to activate py_set_param
packet = dfb.DataPacket('hi')
pipeline25.send(packet)
self.assertEquals(something_filter.header_size, 256)
self.assertEquals(other_filter.header_size, 128)
def test_variable_batch2(self):
# Using Python embedded environment, read batch size from data
config = '''\
[--main--]
ftype = variable_batch
description = Test variable batch passing
##dynamic = true
[py_init_batch_size]
BATCH_SIZE = 2
READING_SIZE = True
[py_read_batch_size]
if READING_SIZE:
BATCH_SIZE = int(packet.data)
READING_SIZE = False
packet.fork_dest = 'branch'
else:
init_batch_size()
[batch]
dynamic = true
size = %BATCH_SIZE
[--route--]
py_init_batch_size >>>
#batch:%BATCH_SIZE >>>
batch >>>
py_read_batch_size >>>
sink
'''
ppln2 = ppln.Pipeline(factory=self.factory, config=config)
source = 'Mary had a little lamb; its fleece was white as snow.'
source2 = ''.join(('%2.2d' % len(x) + x) for x in source.split())
print '**13125** source2 =', source2
packet2 = dfb.DataPacket(source2)
ppln2.send(packet2)
ppln2.shut_down()
print '**13120** results = ', '|'.join(ppln2.getf('sink').all_data)
self.assertEquals(' '.join(ppln2.getf('sink').all_data), source)
print '**13150** finished test_variable_batch2'
def test_filter1(self):
import filterpype.filter_factory as ff
factory = ff.DemoFilterFactory()
config = '''
[--main--]
ftype = demo
description = TO-DO: docstring
[--route--]
batch:3 >>>
reverse_string >>>
sink
'''
pipeline1 = ppln.Pipeline(factory=factory, config=config)
# Manual connection for now
pgf = pipeline1.get_filter
pipeline1.first_filter = pgf('batch')
## pgf('batch').next_filter = pgf('reverse_string')
## pgf('reverse_string').next_filter = pgf('sink')
for string in self.source_data:
pipeline1.send(dfb.DataPacket(string))
print '**1700** about to close'
pipeline1.shut_down()
for part in pgf('sink').results:
print '**2482** ', part.data
## print pgf('sink'].all_data
self.assertEquals(pgf('sink').results[-8].data, '321')
self.assertEquals(pgf('sink').results[-7].data, '654')
self.assertEquals(pgf('sink').results[-6].data, '496')
self.assertEquals(pgf('sink').results[-5].data, '893')
self.assertEquals(pgf('sink').results[-4].data, '167')
self.assertEquals(pgf('sink').results[-3].data, '010')
self.assertEquals(pgf('sink').results[-2].data, '987')
self.assertEquals(pgf('sink').results[-1].data, 'X')
def test_copy_file_obj1(self):
if os.path.exists(self.file_name_out):
os.remove(self.file_name_out)
copy_file_comp_ppln = ppln.CopyFileCompression(
factory=self.factory,
dest_file_name=self.file_name,
callback=mock.Mock())
file_obj = open(self.file_name, 'rb')
copy_file_comp_ppln.send(dfb.DataPacket(file_obj))
copy_file_comp_ppln.shut_down()
# Uncompress file and check its size and content
import bz2
decomp = bz2.BZ2Decompressor()
out_file_fd = open(self.file_name_out, 'r')
new_file_data = bz2.decompress(out_file_fd.read())
out_file_fd.close()
self.assertEquals(len(new_file_data), 27)
self.assertEquals(len(new_file_data), os.path.getsize(self.file_name))
self.assertEquals(new_file_data, self.file_contents)
print "**1835** sizes are equal"
def test_extract_many_attributes(self):
data_in = """
person's name=Postman pat
age=500000
gender=mail
hair col0ur=balding
"""
extractor = ppln.ExtractManyAttributes(factory=self.factory,
attr_delim='equals')
extractor_filter = extractor.getf('attribute_extractor')
sink = df.Sink()
extractor_filter.next_filter = sink
extractor.send(dfb.DataPacket(data_in))
#TEST PACKET NOT FILTER
self.assertEqual(sink.results[0].persons_name, 'Postman pat')
self.assertEqual(sink.results[1].age, '500000')
self.assertEqual(sink.results[2].gender, 'mail')
self.assertEqual(sink.results[3].hair_col0ur, 'balding')
def test_closing_main_and_branch(self):
return # test_closing_main_and_branch TO-DO
self.hidden_branch_filter.send(dfb.DataPacket('hello',
fork_dest='main'))
self.hidden_branch_filter.shut_down()
self.sink_main.send(dfb.DataPacket('world'))
def test_data_length(self):
packet10 = dfb.DataPacket('jane')
self.assertEquals(packet10.data_length, 4)
packet11 = dfb.DataPacket([1,2,3])
self.assertEquals(packet11.data_length, 0)
def test_clear_data(self):
packet9 = dfb.DataPacket('bill', roll=27)
packet9.clear_data()
self.assertEquals(packet9.data, '')
self.assertEquals(packet9.roll, 27)
