def test_extract_many_attributes_split_words(self): data_in = """ TIME CHIP SERIAL IS: 5ABF3. OTHER VALUE IS: 312. SOME TRAILING TEXT """ extractor = ppln.ExtractManyAttributesSplitWords(\ factory=self.factory, split_on_str=". ", attr_delim='col') extractor_filter = extractor.getf('attribute_extractor') sink = df.Sink() extractor_filter.next_filter = sink extractor.send(dfb.DataPacket(data_in)) #TEST PACKET NOT FILTER self.assertEqual(sink.results[0].time_chip_serial_is, '5ABF3') self.assertEqual(sink.results[1].other_value_is, '312')
def setUp(self): self.factory = ff.DemoFilterFactory() self.packet1 = dfb.DataPacket(height=1) self.packet2 = dfb.DataPacket(height=2) self.packet3 = dfb.DataPacket(height=3) self.packet4 = dfb.DataPacket(height=4) self.packet5 = dfb.DataPacket(height=5) self.packet6 = dfb.DataPacket(height=9)
def test_01(self): ##inner_bar = ppln_demo.InnerBar(factory=self.factory, ##temperature=15, ##speed=95) small_pipe_baz = ppln_demo.SmallPipeBaz(factory=self.factory, temperature=15, speed=95) print '\n**14420**', 60 * '-' print '\n'.join(small_pipe_baz.module_loc._python_code_lines) print '**14420**', 60 * '-' packet1 = dfb.DataPacket('hello') small_pipe_baz.send(packet1) new_temp = 34 print '\n**14460** Changing temperature from %d to %d' % ( small_pipe_baz.temperature, new_temp) small_pipe_baz.temperature = new_temp packet2 = dfb.DataPacket('and') small_pipe_baz.send(packet2) new_speed = 700 print '\n**14470** Changing speed from %d to %d' % ( small_pipe_baz.speed, new_speed) small_pipe_baz.speed = new_speed packet3 = dfb.DataPacket('goodbye') small_pipe_baz.send(packet3) new_SPEED = 20 print '\n**14495** Changing SPEED from %s to %s' % ( small_pipe_baz.emb_module.SPEED, new_SPEED) small_pipe_baz.emb_module.SPEED = new_SPEED packet4 = dfb.DataPacket('world') small_pipe_baz.send(packet4) small_pipe_baz.shut_down()
def test_dynamic_pipeline2(self): # Set dynamic=True by calling make_dynamic() pipeline23 = ppln.DynamicPipeline(factory=self.factory, config=self.config1, foo='joe') self.assertEquals(pipeline23.foo, 'joe') self.assertEquals(pipeline23.bar, 27) pipeline23.bar = '%SPEED' # All u/c params should default to dfb.k_unset before packet # is sent through self.assertEquals(pipeline23.bar, dfb.k_unset) # Send packet to activate py_set_param packet = dfb.DataPacket('goodbye') pipeline23.send(packet) self.assertEquals(pipeline23.bar, 170)
def test_dynamic_pipeline1(self): # Set dynamic=True in the kwargs passed in pipeline22 = ppln.DynamicPipeline(factory=self.factory, config=self.config1, foo='jim') self.assertEquals(pipeline22.foo, 'jim') self.assertEquals(pipeline22.bar, 27) pipeline22.bar = '%SPEED' # All u/c params should default to dfb.k_unset before packet # is sent through self.assertEquals(pipeline22.bar, dfb.k_unset) # Send packet to activate py_set_param packet = dfb.DataPacket('hello') pipeline22.send(packet) self.assertEquals(pipeline22.bar, 170)
def setUp(self): self.factory = ff.DemoFilterFactory() self.pipeline1 = ppln.Pipeline(factory=self.factory, config=self.config) self.packet1 = dfb.DataPacket(height=1) self.packet2 = dfb.DataPacket(height=2) self.packet3 = dfb.DataPacket(height=3) self.packet4 = dfb.DataPacket(height=4) self.packet5 = dfb.DataPacket(height=5) self.packet6 = dfb.DataPacket(height=9)
def test_dynamic_filter_in_static_pipeline3(self): # Batch is set to be reversibly dynamic in config config3 = '''\ [--main--] ftype = demo_static_pipeline3 description = Static pipeline with reversible dynamic filter keys = foo, bar:43 #dynamic = true [py_set_param] print '**16130** Setting BATCH_SIZE to 256' BATCH_SIZE_SOMETHING = 256 BATCH_SIZE_OTHER = 128 [distill_something] ftype = distill_header dynamic = true [distill_other] ftype = distill_header dynamic = true [--route--] py_set_param >>> distill_something:%BATCH_SIZE_SOMETHING >>> (pass_through) distill_other:%BATCH_SIZE_OTHER >>> (pass_through) sink ''' pipeline25 = ppln.Pipeline(factory=self.factory, config=config3, foo='jim') self.assertEquals(pipeline25.foo, 'jim') self.assertEquals(pipeline25.bar, 43) something_filter = pipeline25.getf('distill_something') other_filter = pipeline25.getf('distill_other') # Dynamic value not yet set self.assertEquals(something_filter.header_size, dfb.k_unset) # Send packet to activate py_set_param packet = dfb.DataPacket('hi') pipeline25.send(packet) self.assertEquals(something_filter.header_size, 256) self.assertEquals(other_filter.header_size, 128)
def test_variable_batch2(self): # Using Python embedded environment, read batch size from data config = '''\ [--main--] ftype = variable_batch description = Test variable batch passing ##dynamic = true [py_init_batch_size] BATCH_SIZE = 2 READING_SIZE = True [py_read_batch_size] if READING_SIZE: BATCH_SIZE = int(packet.data) READING_SIZE = False packet.fork_dest = 'branch' else: init_batch_size() [batch] dynamic = true size = %BATCH_SIZE [--route--] py_init_batch_size >>> #batch:%BATCH_SIZE >>> batch >>> py_read_batch_size >>> sink ''' ppln2 = ppln.Pipeline(factory=self.factory, config=config) source = 'Mary had a little lamb; its fleece was white as snow.' source2 = ''.join(('%2.2d' % len(x) + x) for x in source.split()) print '**13125** source2 =', source2 packet2 = dfb.DataPacket(source2) ppln2.send(packet2) ppln2.shut_down() print '**13120** results = ', '|'.join(ppln2.getf('sink').all_data) self.assertEquals(' '.join(ppln2.getf('sink').all_data), source) print '**13150** finished test_variable_batch2'
def test_filter1(self): import filterpype.filter_factory as ff factory = ff.DemoFilterFactory() config = ''' [--main--] ftype = demo description = TO-DO: docstring [--route--] batch:3 >>> reverse_string >>> sink ''' pipeline1 = ppln.Pipeline(factory=factory, config=config) # Manual connection for now pgf = pipeline1.get_filter pipeline1.first_filter = pgf('batch') ## pgf('batch').next_filter = pgf('reverse_string') ## pgf('reverse_string').next_filter = pgf('sink') for string in self.source_data: pipeline1.send(dfb.DataPacket(string)) print '**1700** about to close' pipeline1.shut_down() for part in pgf('sink').results: print '**2482** ', part.data ## print pgf('sink'].all_data self.assertEquals(pgf('sink').results[-8].data, '321') self.assertEquals(pgf('sink').results[-7].data, '654') self.assertEquals(pgf('sink').results[-6].data, '496') self.assertEquals(pgf('sink').results[-5].data, '893') self.assertEquals(pgf('sink').results[-4].data, '167') self.assertEquals(pgf('sink').results[-3].data, '010') self.assertEquals(pgf('sink').results[-2].data, '987') self.assertEquals(pgf('sink').results[-1].data, 'X')
def test_copy_file_obj1(self): if os.path.exists(self.file_name_out): os.remove(self.file_name_out) copy_file_comp_ppln = ppln.CopyFileCompression( factory=self.factory, dest_file_name=self.file_name, callback=mock.Mock()) file_obj = open(self.file_name, 'rb') copy_file_comp_ppln.send(dfb.DataPacket(file_obj)) copy_file_comp_ppln.shut_down() # Uncompress file and check its size and content import bz2 decomp = bz2.BZ2Decompressor() out_file_fd = open(self.file_name_out, 'r') new_file_data = bz2.decompress(out_file_fd.read()) out_file_fd.close() self.assertEquals(len(new_file_data), 27) self.assertEquals(len(new_file_data), os.path.getsize(self.file_name)) self.assertEquals(new_file_data, self.file_contents) print "**1835** sizes are equal"
def test_extract_many_attributes(self): data_in = """ person's name=Postman pat age=500000 gender=mail hair col0ur=balding """ extractor = ppln.ExtractManyAttributes(factory=self.factory, attr_delim='equals') extractor_filter = extractor.getf('attribute_extractor') sink = df.Sink() extractor_filter.next_filter = sink extractor.send(dfb.DataPacket(data_in)) #TEST PACKET NOT FILTER self.assertEqual(sink.results[0].persons_name, 'Postman pat') self.assertEqual(sink.results[1].age, '500000') self.assertEqual(sink.results[2].gender, 'mail') self.assertEqual(sink.results[3].hair_col0ur, 'balding')
def test_closing_main_and_branch(self): return # test_closing_main_and_branch TO-DO self.hidden_branch_filter.send(dfb.DataPacket('hello', fork_dest='main')) self.hidden_branch_filter.shut_down() self.sink_main.send(dfb.DataPacket('world'))
def test_data_length(self): packet10 = dfb.DataPacket('jane') self.assertEquals(packet10.data_length, 4) packet11 = dfb.DataPacket([1,2,3]) self.assertEquals(packet11.data_length, 0)
def test_clear_data(self): packet9 = dfb.DataPacket('bill', roll=27) packet9.clear_data() self.assertEquals(packet9.data, '') self.assertEquals(packet9.roll, 27)