def setUp( self ): self.reader = StaticSource( FieldSet([ Field( 'name', StringFieldType() ), Field( 'email', StringFieldType() ) ], FieldMap({ 'name': 0, 'email': 1 })) ) self.reader.setResource([ [ 'Ochala Wild', 'Ochala [email protected]' ], [ 'Sina Venomous', 'Sina [email protected]' ], [ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ], [ 'Sermak Bad', 'Sermak [email protected]' ], [ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ], [ 'Pendus Inhuman', 'Pendus [email protected]' ], [ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ], [ 'Shard Brutal', 'Shard [email protected]' ], [ 'Sina Cruel', 'Sina [email protected]' ], [ 'Deadly Ohmar', 'Deadly [email protected]' ], [ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ], [ 'Calden Frigid', 'Calden [email protected]' ], [ 'Acid Reaper', 'Acid [email protected]' ], [ 'Raven Seth', 'Raven [email protected]' ], [ 'Rivatha Todal', 'Rivatha [email protected]' ], [ 'Panic Oliviaezit', 'Panic [email protected]' ], [ 'Tomara Wild', 'Tomara [email protected]' ], [ 'Venessa Metalhead', 'Venessa [email protected]' ] ])
def test_listexpander_map( self ): static_source = StaticSource( FieldSet([ Field( 'city', StringFieldType() ), Field( 'geometry', ListFieldType() ), Field( 'latitude', FloatFieldType() ), Field( 'longitude', FloatFieldType() ) ], FieldMap({ 'city': 0, 'geometry': 1 })) ) static_source.setResource([ [ 'Balatonlelle', [[17.6874552,46.7871465],[17.6865955,46.7870049],[17.6846158,46.7866786],[17.6834977,46.7864944],[17.6822251,46.7862847],[17.6815319,46.7861705],[17.6811473,46.7861071],[17.6795989,46.785852],[17.6774482,46.7854976],[17.6739061,46.7849139],[17.6729351,46.7847539],[17.6720789,46.7846318]] ], [ 'Balatonlelle', [[17.6871206,46.7880197],[17.6874552,46.7871465]] ], [ 'Balatonboglar', "[[17.6709959,46.7843474],[17.6710995,46.7840514],[17.6711443,46.7838791],[17.6711725,46.7837746],[17.6713146,46.7831483]]" ] ]) expand = ListExpanderExpand( static_source, 'geometry', expanderMap = { 'latitude': 1, 'longitude': 0 } ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), 19 ) self.assertEqual( records[10].getField('city').getValue(), 'Balatonlelle' ) self.assertEqual( records[10].getField('latitude').getValue(), 46.7847539 ) self.assertEqual( records[-1].getField('city').getValue(), 'Balatonboglar' ) self.assertEqual( records[-1].getField('longitude').getValue(), 17.6713146 )
def test_listexpander( self ): static_source = StaticSource( FieldSet([ Field( 'namelist', ListFieldType() ), Field( 'email', StringFieldType() ), Field( 'year', IntegerFieldType() ), Field( 'name', StringFieldType() ) ], FieldMap({ 'namelist': 0, 'email': 1, 'year': 2 })) ) static_source.setResource([ [ list('El Agent'), 'El [email protected]', 2008 ], [ list('Serious Electron'), 'Serious [email protected]', 2008 ], [ list('Brave Wizard'), 'Brave [email protected]', 2008 ] ]) expand = ListExpanderExpand( static_source, 'namelist', 'name' ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), len(list('El Agent')) + len(list('Serious Electron')) + len(list('Brave Wizard')) ) self.assertEqual( records[-1].getField('name').getValue(), 'd' ) self.assertEqual( records[-1].getField('email').getValue(), 'Brave [email protected]' )
def test_joinbykey_modifier( self ): source = StaticSource( FieldSet([ Field( 'email', StringFieldType(), key = True ), Field( 'age', IntegerFieldType() ) ], FieldMap({ 'email': 0, 'age': 1 })) ) source.setResource([ [ 'El [email protected]', 12 ], [ 'Ochala [email protected]', 14 ], [ 'Sina [email protected]', 17 ], [ 'Akassa Savage [email protected]', 16 ], [ 'Sermak [email protected]', 22 ], [ 'Olivia Deadly [email protected]', 32 ], [ '*****@*****.**', 42 ], [ 'Naria Cold-blodded [email protected]', 22 ], [ '*****@*****.**', 54 ], [ 'Sina [email protected]', 56 ], [ 'Deadly [email protected]', 43 ], [ 'Mylenedriz [email protected]', 23 ], [ 'Calden [email protected]', 35 ], [ '*****@*****.**', 56 ], [ 'Raven [email protected]', 23 ], [ '*****@*****.**', 45 ], [ 'Pluto [email protected]', 64 ], [ 'Southern [email protected]', 53 ], [ 'Serious [email protected]', 62 ], [ '*****@*****.**', 63 ], [ 'Risky [email protected]', 21 ], [ 'Rivatha [email protected]', 56 ], [ 'Panic [email protected]', 25 ], [ 'Tomara [email protected]', 46 ], [ 'Venessa [email protected]', 53 ], [ 'Western [email protected]', 71 ], [ '*****@*****.**', 76 ] ]) records = [ r for r in JoinByKeyModifier( self.reader, fieldNames = [ 'age' ], source = source ).initialize().getRecords() ] self.assertEqual( len( records ), 85 ) self.assertIsNone( records[-1].getField('age').getValue() ) self.assertEqual( records[-2].getField('age').getValue(), 71 ) self.assertEqual( records[0].getField('age').getValue(), 12 )
def test_joinbykey_modifier(self): source = StaticSource( FieldSet([ Field('email', StringFieldType(), key=True), Field('age', IntegerFieldType()) ], FieldMap({ 'email': 0, 'age': 1 }))) source.setResource( [['El [email protected]', 12], ['Ochala [email protected]', 14], ['Sina [email protected]', 17], ['Akassa Savage [email protected]', 16], ['Sermak [email protected]', 22], ['Olivia Deadly [email protected]', 32], ['*****@*****.**', 42], ['Naria Cold-blodded [email protected]', 22], ['*****@*****.**', 54], ['Sina [email protected]', 56], ['Deadly [email protected]', 43], ['Mylenedriz [email protected]', 23], ['Calden [email protected]', 35], ['*****@*****.**', 56], ['Raven [email protected]', 23], ['*****@*****.**', 45], ['Pluto [email protected]', 64], ['Southern [email protected]', 53], ['Serious [email protected]', 62], ['*****@*****.**', 63], ['Risky [email protected]', 21], ['Rivatha [email protected]', 56], ['Panic [email protected]', 25], ['Tomara [email protected]', 46], ['Venessa [email protected]', 53], ['Western [email protected]', 71], ['*****@*****.**', 76]]) records = [ r for r in JoinByKeyModifier(self.reader, fieldNames=['age'], source=source).initialize().getRecords() ] self.assertEqual(len(records), 85) self.assertIsNone(records[-1].getField('age').getValue()) self.assertEqual(records[-2].getField('age').getValue(), 71) self.assertEqual(records[0].getField('age').getValue(), 12)
def test_field( self ): static_source = StaticSource( FieldSet([ Field( 'year', IntegerFieldType() ), Field( 'country', StringFieldType() ), Field( 'count', IntegerFieldType() ), Field( 'cz', IntegerFieldType() ), Field( 'hu', IntegerFieldType() ), Field( 'sk', IntegerFieldType() ), Field( 'pl', IntegerFieldType() ) ], FieldMap({ 'year': 0, 'cz': 1, 'hu': 2, 'sk': 3, 'pl': 4 })) ) static_source.setResource([ [1999,32,694,129,230], [1999,395,392,297,453], [1999,635,812,115,97] ]) expand = FieldExpand( static_source, fieldNamesAndLabels = { 'cz': 'Czech', 'hu': 'Hungary', 'sk': 'Slovak', 'pl': 'Poland', }, valueFieldName = 'count', labelFieldName = 'country' ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), 12 ) self.assertEqual( records[-1].getField('year').getValue(), 1999 ) self.assertEqual( records[-1].getField('count').getValue(), 812 ) self.assertEqual( records[-1].getField('pl').getValue(), 97 ) self.assertEqual( records[-1].getField('country').getValue(), 'Hungary' )
def test_melt( self ): static_source = StaticSource( FieldSet([ Field( 'first', StringFieldType() ), Field( 'height', FloatFieldType() ), Field( 'last', StringFieldType() ), Field( 'weight', IntegerFieldType() ), Field( 'iq', IntegerFieldType() ), Field( 'quantity', StringFieldType() ), Field( 'value', FloatFieldType() ) ], FieldMap({ 'first': 0, 'height': 1, 'last': 2, 'weight': 3, 'iq': 4 })) ) static_source.setResource([ ['John',5.5,'Doe',130,102], ['Mary',6.0,'Bo',150,98] ]) expand = MeltExpand( static_source, fieldNames = ['first','last'], valueFieldName = 'value', labelFieldName = 'quantity' ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), 6 ) self.assertEqual( len( records[0].getFieldNames() ), 4) self.assertEqual( records[-1].getField('first').getValue(), 'Mary' ) self.assertEqual( records[-1].getField('last').getValue(), 'Bo' ) self.assertEqual( records[-1].getField('quantity').getValue(), 'height' ) self.assertEqual( records[-1].getField('value').getValue(), 6.0 )
def setUp( self ): self.reader = StaticSource( FieldSet([ Field( 'name', StringFieldType() ), Field( 'email', StringFieldType() ), Field( 'year', IntegerFieldType() ), Field( 'after_year', IntegerFieldType() ), Field( 'age', IntegerFieldType() ) ], FieldMap({ 'name': 0, 'email': 1, 'year': 2, 'after_year': 3 })) ) self.reader.setResource([ [ 'El Agent', 'El [email protected]', 2008, 2008 ], [ 'Serious Electron', 'Serious [email protected]', 2008, 2013 ], [ 'Brave Wizard', 'Brave [email protected]', 2008, 2008 ], [ 'Forgotten Itchy Emperor', 'Forgotten Itchy [email protected]', 2008, 2013 ], [ 'The Moving Monkey', 'The Moving [email protected]', 2008, 2008 ], [ 'Evil Ghostly Brigadier', 'Evil Ghostly [email protected]', 2008, 2013 ], [ 'Strangely Oyster', 'Strangely [email protected]', 2008, 2008 ], [ 'Anaconda Silver', 'Anaconda [email protected]', 2006, 2008 ], [ 'Hawk Tough', 'Hawk [email protected]', 2004, 2008 ], [ 'The Disappointed Craw', 'The Disappointed [email protected]', 2008, 2013 ], [ 'The Raven', 'The [email protected]', 1999, 2008 ], [ 'Ruby Boomerang', 'Ruby [email protected]', 2008, 2008 ], [ 'Skunk Tough', 'Skunk [email protected]', 2010, 2008 ], [ 'The Nervous Forgotten Major', 'The Nervous Forgotten [email protected]', 2008, 2013 ], [ 'Bursting Furious Puppet', 'Bursting Furious [email protected]', 2011, 2008 ], [ 'Neptune Eagle', 'Neptune [email protected]', 2011, 2013 ], [ 'The Skunk', 'The [email protected]', 2008, 2013 ], [ 'Lone Demon', 'Lone [email protected]', 2008, 2008 ], [ 'The Skunk', 'The [email protected]', 1999, 2008 ], [ 'Gamma Serious Spear', 'Gamma Serious [email protected]', 2008, 2008 ], [ 'Sleepy Dirty Sergeant', 'Sleepy Dirty [email protected]', 2008, 2008 ], [ 'Red Monkey', 'Red [email protected]', 2008, 2008 ], [ 'Striking Tiger', 'Striking [email protected]', 2005, 2008 ], [ 'Sliding Demon', 'Sliding [email protected]', 2011, 2008 ], [ 'Lone Commander', 'Lone [email protected]', 2008, 2013 ], [ 'Dragon Insane', 'Dragon [email protected]', 2013, 2013 ], [ 'Demon Skilled', 'Demon [email protected]', 2011, 2004 ], [ 'Vulture Lucky', 'Vulture [email protected]', 2003, 2008 ], [ 'The Ranger', 'The [email protected]', 2013, 2008 ], [ 'Morbid Snake', 'Morbid [email protected]', 2011, 2008 ], [ 'Dancing Skeleton', 'Dancing [email protected]', 2001, 2004 ], [ 'The Psycho', 'The [email protected]', 2005, 2008 ], [ 'Jupiter Rider', 'Jupiter [email protected]', 2011, 2008 ], [ 'Green Dog', 'Green [email protected]', 2011, 2008 ], [ 'Brutal Wild Colonel', 'Brutal Wild [email protected]', 2004, 2008 ], [ 'Random Leader', 'Random [email protected]', 2008, 2008 ], [ 'Pluto Brigadier', 'Pluto [email protected]', 2008, 2004 ], [ 'Southern Kangaroo', 'Southern [email protected]', 2008, 2008 ], [ 'Serious Flea', 'Serious [email protected]', 2001, 2005 ], [ 'Nocturnal Raven', 'Nocturnal [email protected]', 2008, 2004 ], [ 'Risky Flea', 'Risky [email protected]', 2005, 2005 ], [ 'The Corporal', 'The [email protected]', 2013, 2008 ], [ 'The Lucky Barbarian', 'The Lucky [email protected]', 2008, 2008 ], [ 'Rocky Serious Dog', 'Rocky Serious [email protected]', 2008, 2008 ], [ 'The Frozen Guardian', 'The Frozen [email protected]', 2008, 2008 ], [ 'Freaky Frostbite', 'Freaky [email protected]', 2008, 2004 ], [ 'The Tired Raven', 'The Tired [email protected]', 2008, 2008 ], [ 'Disappointed Frostbite', 'Disappointed [email protected]', 2008, 2008 ], [ 'The Craw', 'The [email protected]', 2003, 2008 ], [ 'Gutsy Strangely Chief', 'Gutsy Strangely [email protected]', 2008, 2008 ], [ 'Queen Angry', 'Queen [email protected]', 2008, 2008 ], [ 'Pluto Albatross', 'Pluto [email protected]', 2003, 2008 ], [ 'Endless Invader', 'Endless [email protected]', 2003, 2004 ], [ 'Beta Young Sergeant', 'Beta Young [email protected]', 2008, 2011 ], [ 'The Demon', 'The [email protected]', 2003, 2008 ], [ 'Lone Monkey', 'Lone [email protected]', 2011, 2008 ], [ 'Bursting Electron', 'Bursting [email protected]', 2003, 2010 ], [ 'Gangster Solid', 'Gangster [email protected]', 2005, 2009 ], [ 'The Gladiator', 'The [email protected]', 2001, 2002 ], [ 'Flash Frostbite', 'Flash [email protected]', 2005, 2004 ], [ 'The Rainbow Pluto Demon', 'The Rainbow Pluto [email protected]', 2011, 2013 ], [ 'Poseidon Rider', 'Poseidon [email protected]', 2008, 2006 ], [ 'The Old Alpha Brigadier', 'The Old Alpha [email protected]', 2008, 2008 ], [ 'Rough Anaconda', 'Rough [email protected]', 2001, 2011 ], [ 'Tough Dinosaur', 'Tough [email protected]', 2011, 2010 ], [ 'The Lost Dinosaur', 'The Lost [email protected]', 2008, 2008 ], [ 'The Raven', 'The [email protected]', 2005, 2009 ], [ 'The Agent', 'The [email protected]', 2011, 2008 ], [ 'Brave Scarecrow', 'Brave [email protected]', 2008, 2007 ], [ 'Flash Skeleton', 'Flash [email protected]', 2008, 2006 ], [ 'The Admiral', 'The [email protected]', 1998, 2005 ], [ 'The Tombstone', 'The [email protected]', 2013, 2008 ], [ 'Golden Arrow', 'Golden [email protected]', 2008, 2005 ], [ 'White Guardian', 'White [email protected]', 2011, 2004 ], [ 'The Black Eastern Power', 'The Black Eastern [email protected]', 2008, 2008 ], [ 'Ruthless Soldier', 'Ruthless [email protected]', 2008, 2008 ], [ 'Dirty Clown', 'Dirty [email protected]', 2008, 2008 ], [ 'Alpha Admiral', 'Alpha [email protected]', 2008, 2008 ], [ 'Lightning Major', 'Lightning [email protected]', 2008, 2008 ], [ 'The Rock Demon', 'The Rock [email protected]', 2008, 2001 ], [ 'Wild Tiger', 'Wild [email protected]', 2008, 2001 ], [ 'The Pointless Bandit', 'The Pointless [email protected]', 2008, 2008 ], [ 'The Sergeant', 'The [email protected]', 1998, 2002 ], [ 'Western Ogre', 'Western [email protected]', 1998, 2004 ], [ 'Sergeant Strawberry', 'Sergeant [email protected]', 2006, 2008 ] ])
class Test_Modifier( unittest.TestCase ): def setUp( self ): self.reader = StaticSource( FieldSet([ Field( 'name', StringFieldType() ), Field( 'email', StringFieldType() ), Field( 'year', IntegerFieldType() ), Field( 'after_year', IntegerFieldType() ), Field( 'age', IntegerFieldType() ) ], FieldMap({ 'name': 0, 'email': 1, 'year': 2, 'after_year': 3 })) ) self.reader.setResource([ [ 'El Agent', 'El [email protected]', 2008, 2008 ], [ 'Serious Electron', 'Serious [email protected]', 2008, 2013 ], [ 'Brave Wizard', 'Brave [email protected]', 2008, 2008 ], [ 'Forgotten Itchy Emperor', 'Forgotten Itchy [email protected]', 2008, 2013 ], [ 'The Moving Monkey', 'The Moving [email protected]', 2008, 2008 ], [ 'Evil Ghostly Brigadier', 'Evil Ghostly [email protected]', 2008, 2013 ], [ 'Strangely Oyster', 'Strangely [email protected]', 2008, 2008 ], [ 'Anaconda Silver', 'Anaconda [email protected]', 2006, 2008 ], [ 'Hawk Tough', 'Hawk [email protected]', 2004, 2008 ], [ 'The Disappointed Craw', 'The Disappointed [email protected]', 2008, 2013 ], [ 'The Raven', 'The [email protected]', 1999, 2008 ], [ 'Ruby Boomerang', 'Ruby [email protected]', 2008, 2008 ], [ 'Skunk Tough', 'Skunk [email protected]', 2010, 2008 ], [ 'The Nervous Forgotten Major', 'The Nervous Forgotten [email protected]', 2008, 2013 ], [ 'Bursting Furious Puppet', 'Bursting Furious [email protected]', 2011, 2008 ], [ 'Neptune Eagle', 'Neptune [email protected]', 2011, 2013 ], [ 'The Skunk', 'The [email protected]', 2008, 2013 ], [ 'Lone Demon', 'Lone [email protected]', 2008, 2008 ], [ 'The Skunk', 'The [email protected]', 1999, 2008 ], [ 'Gamma Serious Spear', 'Gamma Serious [email protected]', 2008, 2008 ], [ 'Sleepy Dirty Sergeant', 'Sleepy Dirty [email protected]', 2008, 2008 ], [ 'Red Monkey', 'Red [email protected]', 2008, 2008 ], [ 'Striking Tiger', 'Striking [email protected]', 2005, 2008 ], [ 'Sliding Demon', 'Sliding [email protected]', 2011, 2008 ], [ 'Lone Commander', 'Lone [email protected]', 2008, 2013 ], [ 'Dragon Insane', 'Dragon [email protected]', 2013, 2013 ], [ 'Demon Skilled', 'Demon [email protected]', 2011, 2004 ], [ 'Vulture Lucky', 'Vulture [email protected]', 2003, 2008 ], [ 'The Ranger', 'The [email protected]', 2013, 2008 ], [ 'Morbid Snake', 'Morbid [email protected]', 2011, 2008 ], [ 'Dancing Skeleton', 'Dancing [email protected]', 2001, 2004 ], [ 'The Psycho', 'The [email protected]', 2005, 2008 ], [ 'Jupiter Rider', 'Jupiter [email protected]', 2011, 2008 ], [ 'Green Dog', 'Green [email protected]', 2011, 2008 ], [ 'Brutal Wild Colonel', 'Brutal Wild [email protected]', 2004, 2008 ], [ 'Random Leader', 'Random [email protected]', 2008, 2008 ], [ 'Pluto Brigadier', 'Pluto [email protected]', 2008, 2004 ], [ 'Southern Kangaroo', 'Southern [email protected]', 2008, 2008 ], [ 'Serious Flea', 'Serious [email protected]', 2001, 2005 ], [ 'Nocturnal Raven', 'Nocturnal [email protected]', 2008, 2004 ], [ 'Risky Flea', 'Risky [email protected]', 2005, 2005 ], [ 'The Corporal', 'The [email protected]', 2013, 2008 ], [ 'The Lucky Barbarian', 'The Lucky [email protected]', 2008, 2008 ], [ 'Rocky Serious Dog', 'Rocky Serious [email protected]', 2008, 2008 ], [ 'The Frozen Guardian', 'The Frozen [email protected]', 2008, 2008 ], [ 'Freaky Frostbite', 'Freaky [email protected]', 2008, 2004 ], [ 'The Tired Raven', 'The Tired [email protected]', 2008, 2008 ], [ 'Disappointed Frostbite', 'Disappointed [email protected]', 2008, 2008 ], [ 'The Craw', 'The [email protected]', 2003, 2008 ], [ 'Gutsy Strangely Chief', 'Gutsy Strangely [email protected]', 2008, 2008 ], [ 'Queen Angry', 'Queen [email protected]', 2008, 2008 ], [ 'Pluto Albatross', 'Pluto [email protected]', 2003, 2008 ], [ 'Endless Invader', 'Endless [email protected]', 2003, 2004 ], [ 'Beta Young Sergeant', 'Beta Young [email protected]', 2008, 2011 ], [ 'The Demon', 'The [email protected]', 2003, 2008 ], [ 'Lone Monkey', 'Lone [email protected]', 2011, 2008 ], [ 'Bursting Electron', 'Bursting [email protected]', 2003, 2010 ], [ 'Gangster Solid', 'Gangster [email protected]', 2005, 2009 ], [ 'The Gladiator', 'The [email protected]', 2001, 2002 ], [ 'Flash Frostbite', 'Flash [email protected]', 2005, 2004 ], [ 'The Rainbow Pluto Demon', 'The Rainbow Pluto [email protected]', 2011, 2013 ], [ 'Poseidon Rider', 'Poseidon [email protected]', 2008, 2006 ], [ 'The Old Alpha Brigadier', 'The Old Alpha [email protected]', 2008, 2008 ], [ 'Rough Anaconda', 'Rough [email protected]', 2001, 2011 ], [ 'Tough Dinosaur', 'Tough [email protected]', 2011, 2010 ], [ 'The Lost Dinosaur', 'The Lost [email protected]', 2008, 2008 ], [ 'The Raven', 'The [email protected]', 2005, 2009 ], [ 'The Agent', 'The [email protected]', 2011, 2008 ], [ 'Brave Scarecrow', 'Brave [email protected]', 2008, 2007 ], [ 'Flash Skeleton', 'Flash [email protected]', 2008, 2006 ], [ 'The Admiral', 'The [email protected]', 1998, 2005 ], [ 'The Tombstone', 'The [email protected]', 2013, 2008 ], [ 'Golden Arrow', 'Golden [email protected]', 2008, 2005 ], [ 'White Guardian', 'White [email protected]', 2011, 2004 ], [ 'The Black Eastern Power', 'The Black Eastern [email protected]', 2008, 2008 ], [ 'Ruthless Soldier', 'Ruthless [email protected]', 2008, 2008 ], [ 'Dirty Clown', 'Dirty [email protected]', 2008, 2008 ], [ 'Alpha Admiral', 'Alpha [email protected]', 2008, 2008 ], [ 'Lightning Major', 'Lightning [email protected]', 2008, 2008 ], [ 'The Rock Demon', 'The Rock [email protected]', 2008, 2001 ], [ 'Wild Tiger', 'Wild [email protected]', 2008, 2001 ], [ 'The Pointless Bandit', 'The Pointless [email protected]', 2008, 2008 ], [ 'The Sergeant', 'The [email protected]', 1998, 2002 ], [ 'Western Ogre', 'Western [email protected]', 1998, 2004 ], [ 'Sergeant Strawberry', 'Sergeant [email protected]', 2006, 2008 ] ]) def test_joinbykey_modifier( self ): source = StaticSource( FieldSet([ Field( 'email', StringFieldType(), key = True ), Field( 'age', IntegerFieldType() ) ], FieldMap({ 'email': 0, 'age': 1 })) ) source.setResource([ [ 'El [email protected]', 12 ], [ 'Ochala [email protected]', 14 ], [ 'Sina [email protected]', 17 ], [ 'Akassa Savage [email protected]', 16 ], [ 'Sermak [email protected]', 22 ], [ 'Olivia Deadly [email protected]', 32 ], [ '*****@*****.**', 42 ], [ 'Naria Cold-blodded [email protected]', 22 ], [ '*****@*****.**', 54 ], [ 'Sina [email protected]', 56 ], [ 'Deadly [email protected]', 43 ], [ 'Mylenedriz [email protected]', 23 ], [ 'Calden [email protected]', 35 ], [ '*****@*****.**', 56 ], [ 'Raven [email protected]', 23 ], [ '*****@*****.**', 45 ], [ 'Pluto [email protected]', 64 ], [ 'Southern [email protected]', 53 ], [ 'Serious [email protected]', 62 ], [ '*****@*****.**', 63 ], [ 'Risky [email protected]', 21 ], [ 'Rivatha [email protected]', 56 ], [ 'Panic [email protected]', 25 ], [ 'Tomara [email protected]', 46 ], [ 'Venessa [email protected]', 53 ], [ 'Western [email protected]', 71 ], [ '*****@*****.**', 76 ] ]) records = [ r for r in JoinByKeyModifier( self.reader, fieldNames = [ 'age' ], source = source ).initialize().getRecords() ] self.assertEqual( len( records ), 85 ) self.assertIsNone( records[-1].getField('age').getValue() ) self.assertEqual( records[-2].getField('age').getValue(), 71 ) self.assertEqual( records[0].getField('age').getValue(), 12 ) def test_order_modifier( self ): records = [ r for r in OrderModifier( self.reader, fieldNamesAndOrder = [ { 'year': 'DESC' }, { 'name': 'ASC' } ] ).initialize().getRecords() ] self.assertEqual( len( records ), 85 ) self.assertEqual( records[-1].getField('year').getValue(), 1998 ) self.assertEqual( records[-4].getField('year').getValue(), 1999 ) self.assertEqual( records[0].getField('year').getValue(), 2013 ) self.assertEqual( records[-1].getField('name').getValue(), 'Western Ogre' ) self.assertEqual( records[-2].getField('name').getValue(), 'The Sergeant' ) self.assertEqual( records[-3].getField('name').getValue(), 'The Admiral' ) self.assertEqual( records[-4].getField('name').getValue(), 'The Skunk' ) self.assertEqual( records[0].getField('name').getValue(), 'Dragon Insane' ) def test_transform_field_modifier( self ): records = [ r for r in TransformFieldModifier( self.reader, fieldNames = [ 'email' ], transforms = [ ReplaceByRegexpTransform( regexp = u'[ ]+', to = u'' ) ] ).initialize().getRecords() ] self.assertEqual( len( records ), 85 ) self.assertEqual( records[-1].getField('email').getValue(), '*****@*****.**' ) def test_set_modifier_without_param( self ): records = [ r for r in SetModifier( self.reader, fieldNames = [ 'name' ] ).initialize().getRecords() ] self.assertEqual( len( records ), 85 ) self.assertEqual( len([ r for r in records if r.getField('name').getValue() is not None ]), 0 ) def test_set_modifier_with_sprintf_value( self ): records = [ r for r in SetModifier( self.reader, fieldNames = [ 'email' ], value = '%(name)s <%(email)s>' ).initialize().getRecords() ] self.assertEqual( len( records ), 85 ) self.assertEqual( records[-1].getField('email').getValue(), 'Sergeant Strawberry <Sergeant [email protected]>' ) def test_set_modifier_value( self ): records = [ r for r in SetModifier( self.reader, fieldNames = [ 'name' ], value = 'Same name for all' ).initialize().getRecords() ] self.assertEqual( len( records ), 85 ) self.assertEqual( records[-1].getField('name').getValue(), 'Same name for all' ) def test_set_modifier_fn( self ): def setValue( record, field, scope ): if record.getField('year').getValue() == 2008: return 'Same name for 2008 years' return 'Other name for anything else' fn = setValue records = [ r for r in SetModifier( self.reader, fieldNames = [ 'name' ], fn = fn, ).initialize().getRecords() ] self.assertEqual( len( records ), 85 ) self.assertEqual( records[-1].getField('name').getValue(), 'Other name for anything else' ) self.assertEqual( records[0].getField('name').getValue(), 'Same name for 2008 years' ) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Same name for 2008 years' ]), 43 ) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Other name for anything else' ]), 42 ) def test_set_modifier_source( self ): def setValue( record, field, scope ): return 'Found same email address' \ if record.getField('email').getValue() in [ sr.getField('email').getValue() for sr in scope.getSourceRecords() ] \ else 'Not found same email address' fn = setValue source = StaticSource( FieldSet([ Field( 'name', StringFieldType() ), Field( 'email', StringFieldType() ) ], FieldMap({ 'name': 0, 'email': 1 })) ) source.setResource([ [ 'El Agent', 'El [email protected]' ], [ 'Ochala Wild', 'Ochala [email protected]' ], [ 'Sina Venomous', 'Sina [email protected]' ], [ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ], [ 'Sermak Bad', 'Sermak [email protected]' ], [ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ], [ 'Pendus Inhuman', '*****@*****.**' ], [ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ], [ 'Shard Brutal', '*****@*****.**' ], [ 'Sina Cruel', 'Sina [email protected]' ], [ 'Deadly Ohmar', 'Deadly [email protected]' ], [ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ], [ 'Calden Frigid', 'Calden [email protected]' ], [ 'Acid Reaper', '*****@*****.**' ], [ 'Raven Seth', 'Raven [email protected]' ], [ 'Random Leader', '*****@*****.**' ], [ 'Pluto Brigadier', 'Pluto [email protected]' ], [ 'Southern Kangaroo', 'Southern [email protected]' ], [ 'Serious Flea', 'Serious [email protected]' ], [ 'Nocturnal Raven', '*****@*****.**' ], [ 'Risky Flea', 'Risky [email protected]' ], [ 'Rivatha Todal', 'Rivatha [email protected]' ], [ 'Panic Oliviaezit', 'Panic [email protected]' ], [ 'Tomara Wild', 'Tomara [email protected]' ], [ 'Venessa Metalhead', 'Venessa [email protected]' ], [ 'Western Ogre', 'Western [email protected]' ], [ 'Sergeant Strawberry', '*****@*****.**' ] ]) modifier = SetModifier( self.reader, fieldNames = [ 'name' ], source = source, fn = fn ) modifier.initialize() records = [ r for r in modifier.getRecords() ] modifier.finalize() self.assertEqual( len( records ), 85 ) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Found same email address' ]), 6 ) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Not found same email address' ]), 79 )
def test_set_modifier_source( self ): def setValue( record, field, scope ): return 'Found same email address' \ if record.getField('email').getValue() in [ sr.getField('email').getValue() for sr in scope.getSourceRecords() ] \ else 'Not found same email address' fn = setValue source = StaticSource( FieldSet([ Field( 'name', StringFieldType() ), Field( 'email', StringFieldType() ) ], FieldMap({ 'name': 0, 'email': 1 })) ) source.setResource([ [ 'El Agent', 'El [email protected]' ], [ 'Ochala Wild', 'Ochala [email protected]' ], [ 'Sina Venomous', 'Sina [email protected]' ], [ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ], [ 'Sermak Bad', 'Sermak [email protected]' ], [ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ], [ 'Pendus Inhuman', '*****@*****.**' ], [ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ], [ 'Shard Brutal', '*****@*****.**' ], [ 'Sina Cruel', 'Sina [email protected]' ], [ 'Deadly Ohmar', 'Deadly [email protected]' ], [ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ], [ 'Calden Frigid', 'Calden [email protected]' ], [ 'Acid Reaper', '*****@*****.**' ], [ 'Raven Seth', 'Raven [email protected]' ], [ 'Random Leader', '*****@*****.**' ], [ 'Pluto Brigadier', 'Pluto [email protected]' ], [ 'Southern Kangaroo', 'Southern [email protected]' ], [ 'Serious Flea', 'Serious [email protected]' ], [ 'Nocturnal Raven', '*****@*****.**' ], [ 'Risky Flea', 'Risky [email protected]' ], [ 'Rivatha Todal', 'Rivatha [email protected]' ], [ 'Panic Oliviaezit', 'Panic [email protected]' ], [ 'Tomara Wild', 'Tomara [email protected]' ], [ 'Venessa Metalhead', 'Venessa [email protected]' ], [ 'Western Ogre', 'Western [email protected]' ], [ 'Sergeant Strawberry', '*****@*****.**' ] ]) modifier = SetModifier( self.reader, fieldNames = [ 'name' ], source = source, fn = fn ) modifier.initialize() records = [ r for r in modifier.getRecords() ] modifier.finalize() self.assertEqual( len( records ), 85 ) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Found same email address' ]), 6 ) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Not found same email address' ]), 79 )
def setUp(self): self.reader = StaticSource( FieldSet( [ Field("name", StringFieldType()), Field("email", StringFieldType()), Field("year", IntegerFieldType()), Field("after_year", IntegerFieldType()), Field("number", IntegerFieldType()), ], FieldMap({"name": 0, "email": 1, "year": 2, "after_year": 3}), ) ) self.reader.setResource( [ ["El Agent", "El [email protected]", 2008, 2008], ["Serious Electron", "Serious [email protected]", 2008, 2013], ["Brave Wizard", "Brave [email protected]", 2008, 2008], ["Forgotten Itchy Emperor", "Forgotten Itchy [email protected]", 2008, 2013], ["The Moving Monkey", "The Moving [email protected]", 2008, 2008], ["Evil Ghostly Brigadier", "Evil Ghostly [email protected]", 2008, 2013], ["Strangely Oyster", "Strangely [email protected]", 2008, 2008], ["Anaconda Silver", "Anaconda [email protected]", 2006, 2008], ["Hawk Tough", "Hawk [email protected]", 2004, 2008], ["The Disappointed Craw", "The Disappointed [email protected]", 2008, 2013], ["The Raven", "The [email protected]", 1999, 2008], ["Ruby Boomerang", "Ruby [email protected]", 2008, 2008], ["Skunk Tough", "Skunk [email protected]", 2010, 2008], ["The Nervous Forgotten Major", "The Nervous Forgotten [email protected]", 2008, 2013], ["Bursting Furious Puppet", "Bursting Furious [email protected]", 2011, 2008], ["Neptune Eagle", "Neptune [email protected]", 2011, 2013], ["The Skunk", "The [email protected]", 2008, 2013], ["Lone Demon", "Lone [email protected]", 2008, 2008], ["The Skunk", "The [email protected]", 1999, 2008], ["Gamma Serious Spear", "Gamma Serious [email protected]", 2008, 2008], ["Sleepy Dirty Sergeant", "Sleepy Dirty [email protected]", 2008, 2008], ["Red Monkey", "Red [email protected]", 2008, 2008], ["Striking Tiger", "Striking [email protected]", 2005, 2008], ["Sliding Demon", "Sliding [email protected]", 2011, 2008], ["Lone Commander", "Lone [email protected]", 2008, 2013], ["Dragon Insane", "Dragon [email protected]", 2013, 2013], ["Demon Skilled", "Demon [email protected]", 2011, 2004], ["Vulture Lucky", "Vulture [email protected]", 2003, 2008], ["The Ranger", "The [email protected]", 2013, 2008], ["Morbid Snake", "Morbid [email protected]", 2011, 2008], ["Dancing Skeleton", "Dancing [email protected]", 2001, 2004], ["The Psycho", "The [email protected]", 2005, 2008], ["Jupiter Rider", "Jupiter [email protected]", 2011, 2008], ["Green Dog", "Green [email protected]", 2011, 2008], ["Brutal Wild Colonel", "Brutal Wild [email protected]", 2004, 2008], ["Random Leader", "Random [email protected]", 2008, 2008], ["Pluto Brigadier", "Pluto [email protected]", 2008, 2004], ["Southern Kangaroo", "Southern [email protected]", 2008, 2008], ["Serious Flea", "Serious [email protected]", 2001, 2005], ["Nocturnal Raven", "Nocturnal [email protected]", 2008, 2004], ["Risky Flea", "Risky [email protected]", 2005, 2005], ["The Corporal", "The [email protected]", 2013, 2008], ["The Lucky Barbarian", "The Lucky [email protected]", 2008, 2008], ["Rocky Serious Dog", "Rocky Serious [email protected]", 2008, 2008], ["The Frozen Guardian", "The Frozen [email protected]", 2008, 2008], ["Freaky Frostbite", "Freaky [email protected]", 2008, 2004], ["The Tired Raven", "The Tired [email protected]", 2008, 2008], ["Disappointed Frostbite", "Disappointed [email protected]", 2008, 2008], ["The Craw", "The [email protected]", 2003, 2008], ["Gutsy Strangely Chief", "Gutsy Strangely [email protected]", 2008, 2008], ["Queen Angry", "Queen [email protected]", 2008, 2008], ["Pluto Albatross", "Pluto [email protected]", 2003, 2008], ["Endless Invader", "Endless [email protected]", 2003, 2004], ["Beta Young Sergeant", "Beta Young [email protected]", 2008, 2011], ["The Demon", "The [email protected]", 2003, 2008], ["Lone Monkey", "Lone [email protected]", 2011, 2008], ["Bursting Electron", "Bursting [email protected]", 2003, 2010], ["Gangster Solid", "Gangster [email protected]", 2005, 2009], ["The Gladiator", "The [email protected]", 2001, 2002], ["Flash Frostbite", "Flash [email protected]", 2005, 2004], ["The Rainbow Pluto Demon", "The Rainbow Pluto [email protected]", 2011, 2013], ["Poseidon Rider", "Poseidon [email protected]", 2008, 2006], ["The Old Alpha Brigadier", "The Old Alpha [email protected]", 2008, 2008], ["Rough Anaconda", "Rough [email protected]", 2001, 2011], ["Tough Dinosaur", "Tough [email protected]", 2011, 2010], ["The Lost Dinosaur", "The Lost [email protected]", 2008, 2008], ["The Raven", "The [email protected]", 2005, 2009], ["The Agent", "The [email protected]", 2011, 2008], ["Brave Scarecrow", "Brave [email protected]", 2008, 2007], ["Flash Skeleton", "Flash [email protected]", 2008, 2006], ["The Admiral", "The [email protected]", 1998, 2005], ["The Tombstone", "The [email protected]", 2013, 2008], ["Golden Arrow", "Golden [email protected]", 2008, 2005], ["White Guardian", "White [email protected]", 2011, 2004], ["The Black Eastern Power", "The Black Eastern [email protected]", 2008, 2008], ["Ruthless Soldier", "Ruthless [email protected]", 2008, 2008], ["Dirty Clown", "Dirty [email protected]", 2008, 2008], ["Alpha Admiral", "Alpha [email protected]", 2008, 2008], ["Lightning Major", "Lightning [email protected]", 2008, 2008], ["The Rock Demon", "The Rock [email protected]", 2008, 2001], ["Wild Tiger", "Wild [email protected]", 2008, 2001], ["The Pointless Bandit", "The Pointless [email protected]", 2008, 2008], ["The Sergeant", "The [email protected]", 1998, 2002], ["Western Ogre", "Western [email protected]", 1998, 2004], ["Sergeant Strawberry", "Sergeant [email protected]", 2006, 2008], ] )
def test_set_modifier_source(self): def setValue(record, field, scope): return 'Found same email address' \ if record.getField('email').getValue() in [ sr.getField('email').getValue() for sr in scope.getSourceRecords() ] \ else 'Not found same email address' fn = setValue source = StaticSource( FieldSet([ Field('name', StringFieldType()), Field('email', StringFieldType()) ], FieldMap({ 'name': 0, 'email': 1 }))) source.setResource( [['El Agent', 'El [email protected]'], ['Ochala Wild', 'Ochala [email protected]'], ['Sina Venomous', 'Sina [email protected]'], [ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ], ['Sermak Bad', 'Sermak [email protected]'], ['Olivia Deadly Dawod', 'Olivia Deadly [email protected]'], ['Pendus Inhuman', '*****@*****.**'], [ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ], ['Shard Brutal', '*****@*****.**'], ['Sina Cruel', 'Sina [email protected]'], ['Deadly Ohmar', 'Deadly [email protected]'], [ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ], ['Calden Frigid', 'Calden [email protected]'], ['Acid Reaper', '*****@*****.**'], ['Raven Seth', 'Raven [email protected]'], ['Random Leader', '*****@*****.**'], ['Pluto Brigadier', 'Pluto [email protected]'], ['Southern Kangaroo', 'Southern [email protected]'], ['Serious Flea', 'Serious [email protected]'], ['Nocturnal Raven', '*****@*****.**'], ['Risky Flea', 'Risky [email protected]'], ['Rivatha Todal', 'Rivatha [email protected]'], ['Panic Oliviaezit', 'Panic [email protected]'], ['Tomara Wild', 'Tomara [email protected]'], ['Venessa Metalhead', 'Venessa [email protected]'], ['Western Ogre', 'Western [email protected]'], ['Sergeant Strawberry', '*****@*****.**']]) modifier = SetModifier(self.reader, fieldNames=['name'], source=source, fn=fn) modifier.initialize() records = [r for r in modifier.getRecords()] modifier.finalize() self.assertEqual(len(records), 85) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Found same email address' ]), 6) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Not found same email address' ]), 79)
class Test_Filter( unittest.TestCase ): def setUp( self ): self.reader = StaticSource( FieldSet([ Field( 'name', StringFieldType() ), Field( 'email', StringFieldType() ), Field( 'year', IntegerFieldType() ), Field( 'after_year', IntegerFieldType() ) ], FieldMap({ 'name': 0, 'email': 1, 'year': 2, 'after_year': 3 })) ) self.reader.setResource([ [ 'El Agent', 'El [email protected]', 2008, 2008 ], [ 'Serious Electron', 'Serious [email protected]', 2008, 2013 ], [ 'Brave Wizard', 'Brave [email protected]', 2008, 2008 ], [ 'Forgotten Itchy Emperor', 'Forgotten Itchy [email protected]', 2008, 2013 ], [ 'The Moving Monkey', 'The Moving [email protected]', 2008, 2008 ], [ 'Evil Ghostly Brigadier', 'Evil Ghostly [email protected]', 2008, 2013 ], [ 'Strangely Oyster', 'Strangely [email protected]', 2008, 2008 ], [ 'Anaconda Silver', 'Anaconda [email protected]', 2006, 2008 ], [ 'Hawk Tough', 'Hawk [email protected]', 2004, 2008 ], [ 'The Disappointed Craw', 'The Disappointed [email protected]', 2008, 2013 ], [ 'The Raven', 'The [email protected]', 1999, 2008 ], [ 'Ruby Boomerang', 'Ruby [email protected]', 2008, 2008 ], [ 'Skunk Tough', 'Skunk [email protected]', 2010, 2008 ], [ 'The Nervous Forgotten Major', 'The Nervous Forgotten [email protected]', 2008, 2013 ], [ 'Bursting Furious Puppet', 'Bursting Furious [email protected]', 2011, 2008 ], [ 'Neptune Eagle', 'Neptune [email protected]', 2011, 2013 ], [ 'The Skunk', 'The [email protected]', 2008, 2013 ], [ 'Lone Demon', 'Lone [email protected]', 2008, 2008 ], [ 'The Skunk', 'The [email protected]', 1999, 2008 ], [ 'Gamma Serious Spear', 'Gamma Serious [email protected]', 2008, 2008 ], [ 'Sleepy Dirty Sergeant', 'Sleepy Dirty [email protected]', 2008, 2008 ], [ 'Red Monkey', 'Red [email protected]', 2008, 2008 ], [ 'Striking Tiger', 'Striking [email protected]', 2005, 2008 ], [ 'Sliding Demon', 'Sliding [email protected]', 2011, 2008 ], [ 'Lone Commander', 'Lone [email protected]', 2008, 2013 ], [ 'Dragon Insane', 'Dragon [email protected]', 2013, 2013 ], [ 'Demon Skilled', 'Demon [email protected]', 2011, 2004 ], [ 'Vulture Lucky', 'Vulture [email protected]', 2003, 2008 ], [ 'The Ranger', 'The [email protected]', 2013, 2008 ], [ 'Morbid Snake', 'Morbid [email protected]', 2011, 2008 ], [ 'Dancing Skeleton', 'Dancing [email protected]', 2001, 2004 ], [ 'The Psycho', 'The [email protected]', 2005, 2008 ], [ 'Jupiter Rider', 'Jupiter [email protected]', 2011, 2008 ], [ 'Green Dog', 'Green [email protected]', 2011, 2008 ], [ 'Brutal Wild Colonel', 'Brutal Wild [email protected]', 2004, 2008 ], [ 'Random Leader', 'Random [email protected]', 2008, 2008 ], [ 'Pluto Brigadier', 'Pluto [email protected]', 2008, 2004 ], [ 'Southern Kangaroo', 'Southern [email protected]', 2008, 2008 ], [ 'Serious Flea', 'Serious [email protected]', 2001, 2005 ], [ 'Nocturnal Raven', 'Nocturnal [email protected]', 2008, 2004 ], [ 'Risky Flea', 'Risky [email protected]', 2005, 2005 ], [ 'The Corporal', 'The [email protected]', 2013, 2008 ], [ 'The Lucky Barbarian', 'The Lucky [email protected]', 2008, 2008 ], [ 'Rocky Serious Dog', 'Rocky Serious [email protected]', 2008, 2008 ], [ 'The Frozen Guardian', 'The Frozen [email protected]', 2008, 2008 ], [ 'Freaky Frostbite', 'Freaky [email protected]', 2008, 2004 ], [ 'The Tired Raven', 'The Tired [email protected]', 2008, 2008 ], [ 'Disappointed Frostbite', 'Disappointed [email protected]', 2008, 2008 ], [ 'The Craw', 'The [email protected]', 2003, 2008 ], [ 'Gutsy Strangely Chief', 'Gutsy Strangely [email protected]', 2008, 2008 ], [ 'Queen Angry', 'Queen [email protected]', 2008, 2008 ], [ 'Pluto Albatross', 'Pluto [email protected]', 2003, 2008 ], [ 'Endless Invader', 'Endless [email protected]', 2003, 2004 ], [ 'Beta Young Sergeant', 'Beta Young [email protected]', 2008, 2011 ], [ 'The Demon', 'The [email protected]', 2003, 2008 ], [ 'Lone Monkey', 'Lone [email protected]', 2011, 2008 ], [ 'Bursting Electron', 'Bursting [email protected]', 2003, 2010 ], [ 'Gangster Solid', 'Gangster [email protected]', 2005, 2009 ], [ 'The Gladiator', 'The [email protected]', 2001, 2002 ], [ 'Flash Frostbite', 'Flash [email protected]', 2005, 2004 ], [ 'The Rainbow Pluto Demon', 'The Rainbow Pluto [email protected]', 2011, 2013 ], [ 'Poseidon Rider', 'Poseidon [email protected]', 2008, 2006 ], [ 'The Old Alpha Brigadier', 'The Old Alpha [email protected]', 2008, 2008 ], [ 'Rough Anaconda', 'Rough [email protected]', 2001, 2011 ], [ 'Tough Dinosaur', 'Tough [email protected]', 2011, 2010 ], [ 'The Lost Dinosaur', 'The Lost [email protected]', 2008, 2008 ], [ 'The Raven', 'The [email protected]', 2005, 2009 ], [ 'The Agent', 'The [email protected]', 2011, 2008 ], [ 'Brave Scarecrow', 'Brave [email protected]', 2008, 2007 ], [ 'Flash Skeleton', 'Flash [email protected]', 2008, 2006 ], [ 'The Admiral', 'The [email protected]', 1998, 2005 ], [ 'The Tombstone', 'The [email protected]', 2013, 2008 ], [ 'Golden Arrow', 'Golden [email protected]', 2008, 2005 ], [ 'White Guardian', 'White [email protected]', 2011, 2004 ], [ 'The Black Eastern Power', 'The Black Eastern [email protected]', 2008, 2008 ], [ 'Ruthless Soldier', 'Ruthless [email protected]', 2008, 2008 ], [ 'Dirty Clown', 'Dirty [email protected]', 2008, 2008 ], [ 'Alpha Admiral', 'Alpha [email protected]', 2008, 2008 ], [ 'Lightning Major', 'Lightning [email protected]', 2008, 2008 ], [ 'The Rock Demon', 'The Rock [email protected]', 2008, 2001 ], [ 'Wild Tiger', 'Wild [email protected]', 2008, 2001 ], [ 'The Pointless Bandit', 'The Pointless [email protected]', 2008, 2008 ], [ 'The Sergeant', 'The [email protected]', 1998, 2002 ], [ 'Western Ogre', 'Western [email protected]', 1998, 2004 ], [ 'Sergeant Strawberry', 'Sergeant [email protected]', 2008, 2008 ] ]) self.other_source = StaticSource( FieldSet([ Field( 'name', StringFieldType() ), Field( 'email', StringFieldType() ) ], FieldMap({ 'name': 0, 'email': 1 })) ) self.other_source.setResource([ [ 'Ochala Wild', 'Ochala [email protected]' ], [ 'Sina Venomous', 'Sina [email protected]' ], [ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ], [ 'Sermak Bad', 'Sermak [email protected]' ], [ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ], [ 'Pendus Inhuman', '*****@*****.**' ], [ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ], [ 'Shard Brutal', '*****@*****.**' ], [ 'Sina Cruel', 'Sina [email protected]' ], [ 'Deadly Ohmar', 'Deadly [email protected]' ], [ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ], [ 'Calden Frigid', 'Calden [email protected]' ], [ 'Acid Reaper', '*****@*****.**' ], [ 'Raven Seth', 'Raven [email protected]' ], [ 'Random Leader', '*****@*****.**' ], [ 'Pluto Brigadier', 'Pluto [email protected]' ], [ 'Southern Kangaroo', 'Southern [email protected]' ], [ 'Serious Flea', 'Serious [email protected]' ], [ 'Nocturnal Raven', '*****@*****.**' ], [ 'Risky Flea', 'Risky [email protected]' ], [ 'Rivatha Todal', 'Rivatha [email protected]' ], [ 'Panic Oliviaezit', 'Panic [email protected]' ], [ 'Tomara Wild', 'Tomara [email protected]' ], [ 'Venessa Metalhead', 'Venessa [email protected]' ], [ 'Western Ogre', 'Western [email protected]' ], [ 'Sergeant Strawberry', 'Sergeant [email protected]' ] ]) def test_drop_by_condition_filter( self ): records = [ r for r in DropByConditionFilter( self.reader, condition = IsEqualCondition( 2008 ), fieldNames = 'year' ).initialize().getRecords() ] self.assertEqual( len( records ), 41 ) self.assertEqual( records[-1].getField('name').getValue(), 'Western Ogre' ) def test_drop_by_condition_filter_and( self ): records = [ r for r in DropByConditionFilter( self.reader, condition = IsEqualCondition( 2008 ), fieldNames = [ 'year', 'after_year' ] ).initialize().getRecords() ] self.assertEqual( len( records ), 58 ) self.assertEqual( records[-1].getField('name').getValue(), 'Western Ogre' ) def test_drop_by_condition_filter_or( self ): records = [ r for r in DropByConditionFilter( self.reader, condition = IsEqualCondition( 2008 ), fieldNames = [ 'year', 'after_year' ], operation = 'OR' ).initialize().getRecords() ] self.assertEqual( len( records ), 19 ) self.assertEqual( records[-1].getField('name').getValue(), 'Western Ogre' ) def test_keep_by_condition_filter( self ): records = [ r for r in KeepByConditionFilter( self.reader, condition = IsEqualCondition( 2008 ), fieldNames = 'year' ).initialize().getRecords() ] self.assertEqual( len( records ), 44 ) self.assertEqual( records[-1].getField('name').getValue(), 'Sergeant Strawberry' ) def test_keep_by_condition_filter_and( self ): records = [ r for r in KeepByConditionFilter( self.reader, condition = IsEqualCondition( 2008 ), fieldNames = [ 'year', 'after_year' ] ).initialize().getRecords() ] self.assertEqual( len( records ), 27 ) self.assertEqual( records[-1].getField('name').getValue(), 'Sergeant Strawberry' ) def test_keep_by_condition_filter_or( self ): records = [ r for r in KeepByConditionFilter( self.reader, condition = IsEqualCondition( 2008 ), fieldNames = [ 'year', 'after_year' ], operation = 'OR' ).initialize().getRecords() ] self.assertEqual( len( records ), 66 ) self.assertEqual( records[-1].getField('name').getValue(), 'Sergeant Strawberry' ) def test_drop_field_filter( self ): records = [ r for r in DropFieldFilter( self.reader, fieldNames = [ 'email', 'after_year' ] ).initialize().getRecords() ] sourceRecords = [ r for r in self.reader.getRecords() ] self.assertEqual( len( sourceRecords ), len( records ) ) self.assertEqual( len( records[0].getFieldNames() ), 2 ) self.assertEqual( records[0].getFieldNames()[-1], 'year' ) def test_drop_by_source_filter( self ): f = DropBySourceFilter( self.reader, self.other_source, condition = IsEqualCondition('email'), join = 'name' ) f.initialize() records = [ r for r in f.getRecords() ] f.finalize() self.assertEqual( len( records ), 2 ) self.assertEqual( records[-1].getField('name').getValue(), 'Nocturnal Raven' )
class Test_Expand( unittest.TestCase ): def setUp( self ): self.reader = StaticSource( FieldSet([ Field( 'name', StringFieldType() ), Field( 'email', StringFieldType() ) ], FieldMap({ 'name': 0, 'email': 1 })) ) self.reader.setResource([ [ 'Ochala Wild', 'Ochala [email protected]' ], [ 'Sina Venomous', 'Sina [email protected]' ], [ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ], [ 'Sermak Bad', 'Sermak [email protected]' ], [ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ], [ 'Pendus Inhuman', 'Pendus [email protected]' ], [ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ], [ 'Shard Brutal', 'Shard [email protected]' ], [ 'Sina Cruel', 'Sina [email protected]' ], [ 'Deadly Ohmar', 'Deadly [email protected]' ], [ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ], [ 'Calden Frigid', 'Calden [email protected]' ], [ 'Acid Reaper', 'Acid [email protected]' ], [ 'Raven Seth', 'Raven [email protected]' ], [ 'Rivatha Todal', 'Rivatha [email protected]' ], [ 'Panic Oliviaezit', 'Panic [email protected]' ], [ 'Tomara Wild', 'Tomara [email protected]' ], [ 'Venessa Metalhead', 'Venessa [email protected]' ] ]) def test_append_by_source( self ): static_source = StaticSource( FieldSet([ Field( 'name', StringFieldType() ), Field( 'email', StringFieldType() ), Field( 'year', IntegerFieldType() ) ], FieldMap({ 'name': 0, 'email': 1, 'year': 2 })) ) static_source.setResource([ [ 'El Agent', 'El [email protected]', 2008 ], [ 'Serious Electron', 'Serious [email protected]', 2008 ], [ 'Brave Wizard', 'Brave [email protected]', 2008 ], [ 'Forgotten Itchy Emperor', 'Forgotten Itchy [email protected]', 2008 ], [ 'The Moving Monkey', 'The Moving [email protected]', 2008 ], [ 'Evil Ghostly Brigadier', 'Evil Ghostly [email protected]', 2008 ], [ 'Strangely Oyster', 'Strangely [email protected]', 2008 ], [ 'Anaconda Silver', 'Anaconda [email protected]', 2006 ], [ 'Hawk Tough', 'Hawk [email protected]', 2004 ], [ 'The Disappointed Craw', 'The Disappointed [email protected]', 2008 ], [ 'The Raven', 'The [email protected]', 1999 ], [ 'Ruby Boomerang', 'Ruby [email protected]', 2008 ], [ 'Skunk Tough', 'Skunk [email protected]', 2010 ], [ 'The Nervous Forgotten Major', 'The Nervous Forgotten [email protected]', 2008 ], [ 'Bursting Furious Puppet', 'Bursting Furious [email protected]', 2011 ], [ 'Neptune Eagle', 'Neptune [email protected]', 2011 ], [ 'The Skunk', 'The [email protected]', 2008 ], [ 'Lone Demon', 'Lone [email protected]', 2008 ], [ 'The Skunk', 'The [email protected]', 1999 ], [ 'Gamma Serious Spear', 'Gamma Serious [email protected]', 2008 ], [ 'Sleepy Dirty Sergeant', 'Sleepy Dirty [email protected]', 2008 ], [ 'Red Monkey', 'Red [email protected]', 2008 ], [ 'Striking Tiger', 'Striking [email protected]', 2005 ], [ 'Sliding Demon', 'Sliding [email protected]', 2011 ], [ 'Lone Commander', 'Lone [email protected]', 2008 ], [ 'Dragon Insane', 'Dragon [email protected]', 2013 ], [ 'Demon Skilled', 'Demon [email protected]', 2011 ], [ 'Vulture Lucky', 'Vulture [email protected]', 2003 ], [ 'The Ranger', 'The [email protected]', 2013 ], [ 'Morbid Snake', 'Morbid [email protected]', 2011 ], [ 'Dancing Skeleton', 'Dancing [email protected]', 2001 ], [ 'The Psycho', 'The [email protected]', 2005 ], [ 'Jupiter Rider', 'Jupiter [email protected]', 2011 ], [ 'Green Dog', 'Green [email protected]', 2011 ], [ 'Brutal Wild Colonel', 'Brutal Wild [email protected]', 2008 ], [ 'Random Leader', 'Random [email protected]', 2008 ], [ 'Pluto Brigadier', 'Pluto [email protected]', 2008 ], [ 'Southern Kangaroo', 'Southern [email protected]', 2008 ], [ 'Serious Flea', 'Serious [email protected]', 2001 ], [ 'Nocturnal Raven', 'Nocturnal [email protected]', 2008 ], [ 'Risky Flea', 'Risky [email protected]', 2005 ], [ 'The Corporal', 'The [email protected]', 2013 ], [ 'The Lucky Barbarian', 'The Lucky [email protected]', 2008 ], [ 'Rocky Serious Dog', 'Rocky Serious [email protected]', 2008 ], [ 'The Frozen Guardian', 'The Frozen [email protected]', 2008 ], [ 'Freaky Frostbite', 'Freaky [email protected]', 2008 ], [ 'The Tired Raven', 'The Tired [email protected]', 2008 ], [ 'Disappointed Frostbite', 'Disappointed [email protected]', 2008 ], [ 'The Craw', 'The [email protected]', 2003 ], [ 'Gutsy Strangely Chief', 'Gutsy Strangely [email protected]', 2008 ], [ 'Queen Angry', 'Queen [email protected]', 2008 ], [ 'Pluto Albatross', 'Pluto [email protected]', 2003 ], [ 'Endless Invader', 'Endless [email protected]', 2003 ], [ 'Beta Young Sergeant', 'Beta Young [email protected]', 2008 ], [ 'The Demon', 'The [email protected]', 2003 ], [ 'Lone Monkey', 'Lone [email protected]', 2011 ], [ 'Bursting Electron', 'Bursting [email protected]', 2003 ], [ 'Gangster Solid', 'Gangster [email protected]', 2005 ], [ 'The Gladiator', 'The [email protected]', 2001 ], [ 'Flash Frostbite', 'Flash [email protected]', 2005 ], [ 'The Rainbow Pluto Demon', 'The Rainbow Pluto [email protected]', 2011 ], [ 'Poseidon Rider', 'Poseidon [email protected]', 2008 ], [ 'The Old Alpha Brigadier', 'The Old Alpha [email protected]', 2008 ], [ 'Rough Anaconda', 'Rough [email protected]', 2001 ], [ 'Tough Dinosaur', 'Tough [email protected]', 2011 ], [ 'The Lost Dinosaur', 'The Lost [email protected]', 2008 ], [ 'The Raven', 'The [email protected]', 2005 ], [ 'The Agent', 'The [email protected]', 2011 ], [ 'Brave Scarecrow', 'Brave [email protected]', 2008 ], [ 'Flash Skeleton', 'Flash [email protected]', 2008 ], [ 'The Admiral', 'The [email protected]', 1998 ], [ 'The Tombstone', 'The [email protected]', 2013 ], [ 'Golden Arrow', 'Golden [email protected]', 2008 ], [ 'White Guardian', 'White [email protected]', 2011 ], [ 'The Black Eastern Power', 'The Black Eastern [email protected]', 2008 ], [ 'Ruthless Soldier', 'Ruthless [email protected]', 2008 ], [ 'Dirty Clown', 'Dirty [email protected]', 2008 ], [ 'Alpha Admiral', 'Alpha [email protected]', 2008 ], [ 'Lightning Major', 'Lightning [email protected]', 2008 ], [ 'The Rock Demon', 'The Rock [email protected]', 2008 ], [ 'Wild Tiger', 'Wild [email protected]', 2008 ], [ 'The Pointless Bandit', 'The Pointless [email protected]', 2008 ], [ 'The Sergeant', 'The [email protected]', 1998 ], [ 'Western Ogre', 'Western [email protected]', 1998 ], [ 'Sergeant Strawberry', 'Sergeant [email protected]', 2008 ] ]) expand = AppendBySourceExpand( self.reader, static_source ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), 103 ) self.assertEqual( len( records[0].getFieldNames() ), 2 ) self.assertEqual( records[-1].getField('name').getValue(), 'Sergeant Strawberry' ) def test_append( self ): source = CSVSource( FieldSet( fields = [ Field( 'NUMBERS', StringFieldType() ) ], fieldmap = FieldMap({ 'NUMBERS': 0 }) ) ) source.setResource('tests/test_sources/test_csv_append.csv') expand = AppendExpand( source, resource = 'tests/test_sources/test_csv_appended.csv' ) expand.initialize() records = [ r for r in expand.getRecords() ] expand.finalize() self.assertEqual( len( records ), 6 ) self.assertEqual( records[0].getField('NUMBERS').getValue(), 'First' ) self.assertEqual( records[-1].getField('NUMBERS').getValue(), 'Sixth' ) def test_listexpander( self ): static_source = StaticSource( FieldSet([ Field( 'namelist', ListFieldType() ), Field( 'email', StringFieldType() ), Field( 'year', IntegerFieldType() ), Field( 'name', StringFieldType() ) ], FieldMap({ 'namelist': 0, 'email': 1, 'year': 2 })) ) static_source.setResource([ [ list('El Agent'), 'El [email protected]', 2008 ], [ list('Serious Electron'), 'Serious [email protected]', 2008 ], [ list('Brave Wizard'), 'Brave [email protected]', 2008 ] ]) expand = ListExpanderExpand( static_source, 'namelist', 'name' ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), len(list('El Agent')) + len(list('Serious Electron')) + len(list('Brave Wizard')) ) self.assertEqual( records[-1].getField('name').getValue(), 'd' ) self.assertEqual( records[-1].getField('email').getValue(), 'Brave [email protected]' ) def test_field( self ): static_source = StaticSource( FieldSet([ Field( 'year', IntegerFieldType() ), Field( 'country', StringFieldType() ), Field( 'count', IntegerFieldType() ), Field( 'cz', IntegerFieldType() ), Field( 'hu', IntegerFieldType() ), Field( 'sk', IntegerFieldType() ), Field( 'pl', IntegerFieldType() ) ], FieldMap({ 'year': 0, 'cz': 1, 'hu': 2, 'sk': 3, 'pl': 4 })) ) static_source.setResource([ [1999,32,694,129,230], [1999,395,392,297,453], [1999,635,812,115,97] ]) expand = FieldExpand( static_source, fieldNamesAndLabels = { 'cz': 'Czech', 'hu': 'Hungary', 'sk': 'Slovak', 'pl': 'Poland', }, valueFieldName = 'count', labelFieldName = 'country' ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), 12 ) self.assertEqual( records[-1].getField('year').getValue(), 1999 ) self.assertEqual( records[-1].getField('count').getValue(), 812 ) self.assertEqual( records[-1].getField('pl').getValue(), 97 ) self.assertEqual( records[-1].getField('country').getValue(), 'Hungary' ) def test_melt( self ): static_source = StaticSource( FieldSet([ Field( 'first', StringFieldType() ), Field( 'height', FloatFieldType() ), Field( 'last', StringFieldType() ), Field( 'weight', IntegerFieldType() ), Field( 'iq', IntegerFieldType() ), Field( 'quantity', StringFieldType() ), Field( 'value', FloatFieldType() ) ], FieldMap({ 'first': 0, 'height': 1, 'last': 2, 'weight': 3, 'iq': 4 })) ) static_source.setResource([ ['John',5.5,'Doe',130,102], ['Mary',6.0,'Bo',150,98] ]) expand = MeltExpand( static_source, fieldNames = ['first','last'], valueFieldName = 'value', labelFieldName = 'quantity' ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), 6 ) self.assertEqual( len( records[0].getFieldNames() ), 4) self.assertEqual( records[-1].getField('first').getValue(), 'Mary' ) self.assertEqual( records[-1].getField('last').getValue(), 'Bo' ) self.assertEqual( records[-1].getField('quantity').getValue(), 'height' ) self.assertEqual( records[-1].getField('value').getValue(), 6.0 ) def test_listexpander_map( self ): static_source = StaticSource( FieldSet([ Field( 'city', StringFieldType() ), Field( 'geometry', ListFieldType() ), Field( 'latitude', FloatFieldType() ), Field( 'longitude', FloatFieldType() ) ], FieldMap({ 'city': 0, 'geometry': 1 })) ) static_source.setResource([ [ 'Balatonlelle', [[17.6874552,46.7871465],[17.6865955,46.7870049],[17.6846158,46.7866786],[17.6834977,46.7864944],[17.6822251,46.7862847],[17.6815319,46.7861705],[17.6811473,46.7861071],[17.6795989,46.785852],[17.6774482,46.7854976],[17.6739061,46.7849139],[17.6729351,46.7847539],[17.6720789,46.7846318]] ], [ 'Balatonlelle', [[17.6871206,46.7880197],[17.6874552,46.7871465]] ], [ 'Balatonboglar', "[[17.6709959,46.7843474],[17.6710995,46.7840514],[17.6711443,46.7838791],[17.6711725,46.7837746],[17.6713146,46.7831483]]" ] ]) expand = ListExpanderExpand( static_source, 'geometry', expanderMap = { 'latitude': 1, 'longitude': 0 } ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), 19 ) self.assertEqual( records[10].getField('city').getValue(), 'Balatonlelle' ) self.assertEqual( records[10].getField('latitude').getValue(), 46.7847539 ) self.assertEqual( records[-1].getField('city').getValue(), 'Balatonboglar' ) self.assertEqual( records[-1].getField('longitude').getValue(), 17.6713146 )
class Test_Filter(unittest.TestCase): def setUp(self): self.reader = StaticSource( FieldSet([ Field('name', StringFieldType()), Field('email', StringFieldType()), Field('year', IntegerFieldType()), Field('after_year', IntegerFieldType()) ], FieldMap({ 'name': 0, 'email': 1, 'year': 2, 'after_year': 3 }))) self.reader.setResource( [['El Agent', 'El [email protected]', 2008, 2008], [ 'Serious Electron', 'Serious [email protected]', 2008, 2013 ], ['Brave Wizard', 'Brave [email protected]', 2008, 2008], [ 'Forgotten Itchy Emperor', 'Forgotten Itchy [email protected]', 2008, 2013 ], [ 'The Moving Monkey', 'The Moving [email protected]', 2008, 2008 ], [ 'Evil Ghostly Brigadier', 'Evil Ghostly [email protected]', 2008, 2013 ], [ 'Strangely Oyster', 'Strangely [email protected]', 2008, 2008 ], [ 'Anaconda Silver', 'Anaconda [email protected]', 2006, 2008 ], ['Hawk Tough', 'Hawk [email protected]', 2004, 2008], [ 'The Disappointed Craw', 'The Disappointed [email protected]', 2008, 2013 ], ['The Raven', 'The [email protected]', 1999, 2008], [ 'Ruby Boomerang', 'Ruby [email protected]', 2008, 2008 ], ['Skunk Tough', 'Skunk [email protected]', 2010, 2008], [ 'The Nervous Forgotten Major', 'The Nervous Forgotten [email protected]', 2008, 2013 ], [ 'Bursting Furious Puppet', 'Bursting Furious [email protected]', 2011, 2008 ], ['Neptune Eagle', 'Neptune [email protected]', 2011, 2013], ['The Skunk', 'The [email protected]', 2008, 2013], ['Lone Demon', 'Lone [email protected]', 2008, 2008], ['The Skunk', 'The [email protected]', 1999, 2008], [ 'Gamma Serious Spear', 'Gamma Serious [email protected]', 2008, 2008 ], [ 'Sleepy Dirty Sergeant', 'Sleepy Dirty [email protected]', 2008, 2008 ], ['Red Monkey', 'Red [email protected]', 2008, 2008], [ 'Striking Tiger', 'Striking [email protected]', 2005, 2008 ], ['Sliding Demon', 'Sliding [email protected]', 2011, 2008], [ 'Lone Commander', 'Lone [email protected]', 2008, 2013 ], ['Dragon Insane', 'Dragon [email protected]', 2013, 2013], ['Demon Skilled', 'Demon [email protected]', 2011, 2004], ['Vulture Lucky', 'Vulture [email protected]', 2003, 2008], ['The Ranger', 'The [email protected]', 2013, 2008], ['Morbid Snake', 'Morbid [email protected]', 2011, 2008], [ 'Dancing Skeleton', 'Dancing [email protected]', 2001, 2004 ], ['The Psycho', 'The [email protected]', 2005, 2008], ['Jupiter Rider', 'Jupiter [email protected]', 2011, 2008], ['Green Dog', 'Green [email protected]', 2011, 2008], [ 'Brutal Wild Colonel', 'Brutal Wild [email protected]', 2004, 2008 ], ['Random Leader', 'Random [email protected]', 2008, 2008], [ 'Pluto Brigadier', 'Pluto [email protected]', 2008, 2004 ], [ 'Southern Kangaroo', 'Southern [email protected]', 2008, 2008 ], ['Serious Flea', 'Serious [email protected]', 2001, 2005], [ 'Nocturnal Raven', 'Nocturnal [email protected]', 2008, 2004 ], ['Risky Flea', 'Risky [email protected]', 2005, 2005], ['The Corporal', 'The [email protected]', 2013, 2008], [ 'The Lucky Barbarian', 'The Lucky [email protected]', 2008, 2008 ], [ 'Rocky Serious Dog', 'Rocky Serious [email protected]', 2008, 2008 ], [ 'The Frozen Guardian', 'The Frozen [email protected]', 2008, 2008 ], [ 'Freaky Frostbite', 'Freaky [email protected]', 2008, 2004 ], [ 'The Tired Raven', 'The Tired [email protected]', 2008, 2008 ], [ 'Disappointed Frostbite', 'Disappointed [email protected]', 2008, 2008 ], ['The Craw', 'The [email protected]', 2003, 2008], [ 'Gutsy Strangely Chief', 'Gutsy Strangely [email protected]', 2008, 2008 ], ['Queen Angry', 'Queen [email protected]', 2008, 2008], [ 'Pluto Albatross', 'Pluto [email protected]', 2003, 2008 ], [ 'Endless Invader', 'Endless [email protected]', 2003, 2004 ], [ 'Beta Young Sergeant', 'Beta Young [email protected]', 2008, 2011 ], ['The Demon', 'The [email protected]', 2003, 2008], ['Lone Monkey', 'Lone [email protected]', 2011, 2008], [ 'Bursting Electron', 'Bursting [email protected]', 2003, 2010 ], [ 'Gangster Solid', 'Gangster [email protected]', 2005, 2009 ], ['The Gladiator', 'The [email protected]', 2001, 2002], [ 'Flash Frostbite', 'Flash [email protected]', 2005, 2004 ], [ 'The Rainbow Pluto Demon', 'The Rainbow Pluto [email protected]', 2011, 2013 ], [ 'Poseidon Rider', 'Poseidon [email protected]', 2008, 2006 ], [ 'The Old Alpha Brigadier', 'The Old Alpha [email protected]', 2008, 2008 ], [ 'Rough Anaconda', 'Rough [email protected]', 2001, 2011 ], [ 'Tough Dinosaur', 'Tough [email protected]', 2011, 2010 ], [ 'The Lost Dinosaur', 'The Lost [email protected]', 2008, 2008 ], ['The Raven', 'The [email protected]', 2005, 2009], ['The Agent', 'The [email protected]', 2011, 2008], [ 'Brave Scarecrow', 'Brave [email protected]', 2008, 2007 ], [ 'Flash Skeleton', 'Flash [email protected]', 2008, 2006 ], ['The Admiral', 'The [email protected]', 1998, 2005], ['The Tombstone', 'The [email protected]', 2013, 2008], ['Golden Arrow', 'Golden [email protected]', 2008, 2005], [ 'White Guardian', 'White [email protected]', 2011, 2004 ], [ 'The Black Eastern Power', 'The Black Eastern [email protected]', 2008, 2008 ], [ 'Ruthless Soldier', 'Ruthless [email protected]', 2008, 2008 ], ['Dirty Clown', 'Dirty [email protected]', 2008, 2008], ['Alpha Admiral', 'Alpha [email protected]', 2008, 2008], [ 'Lightning Major', 'Lightning [email protected]', 2008, 2008 ], [ 'The Rock Demon', 'The Rock [email protected]', 2008, 2001 ], ['Wild Tiger', 'Wild [email protected]', 2008, 2001], [ 'The Pointless Bandit', 'The Pointless [email protected]', 2008, 2008 ], ['The Sergeant', 'The [email protected]', 1998, 2002], ['Western Ogre', 'Western [email protected]', 1998, 2004], [ 'Sergeant Strawberry', 'Sergeant [email protected]', 2008, 2008 ]]) self.other_source = StaticSource( FieldSet([ Field('name', StringFieldType()), Field('email', StringFieldType()) ], FieldMap({ 'name': 0, 'email': 1 }))) self.other_source.setResource( [['Ochala Wild', 'Ochala [email protected]'], ['Sina Venomous', 'Sina [email protected]'], [ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ], ['Sermak Bad', 'Sermak [email protected]'], ['Olivia Deadly Dawod', 'Olivia Deadly [email protected]'], ['Pendus Inhuman', '*****@*****.**'], [ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ], ['Shard Brutal', '*****@*****.**'], ['Sina Cruel', 'Sina [email protected]'], ['Deadly Ohmar', 'Deadly [email protected]'], [ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ], ['Calden Frigid', 'Calden [email protected]'], ['Acid Reaper', '*****@*****.**'], ['Raven Seth', 'Raven [email protected]'], ['Random Leader', '*****@*****.**'], ['Pluto Brigadier', 'Pluto [email protected]'], ['Southern Kangaroo', 'Southern [email protected]'], ['Serious Flea', 'Serious [email protected]'], ['Nocturnal Raven', '*****@*****.**'], ['Risky Flea', 'Risky [email protected]'], ['Rivatha Todal', 'Rivatha [email protected]'], ['Panic Oliviaezit', 'Panic [email protected]'], ['Tomara Wild', 'Tomara [email protected]'], ['Venessa Metalhead', 'Venessa [email protected]'], ['Western Ogre', 'Western [email protected]'], ['Sergeant Strawberry', 'Sergeant [email protected]']]) def test_logger_filter_default(self): filtr = DropByConditionFilter(self.reader, condition=IsEqualCondition(2008), fieldNames='year') filtr.setLogFile(logFile='tests/logger', appendLog=False, logger=None) filtr.initialize() records = [r for r in filtr.getRecords()] filtr.finalize() hashcode = hashlib.md5(open('tests/logger', 'rb').read()).hexdigest() os.unlink('tests/logger') self.assertEqual(hashcode, '95eff99740e2d8d2b50808357e006293') def test_logger_filter_fn(self): filtr = DropByConditionFilter(self.reader, condition=IsEqualCondition(2008), fieldNames='year') filtr.setLogFile(logFile='tests/logger', appendLog=False, logger='tests.test_filter.notDroppedLogger') filtr.initialize() records = [r for r in filtr.getRecords()] filtr.finalize() hashcode = hashlib.md5(open('tests/logger', 'rb').read()).hexdigest() os.unlink('tests/logger') self.assertEqual(hashcode, '3c30fe720f1b410fce6cf17651978b13') def test_drop_by_condition_filter(self): records = [ r for r in DropByConditionFilter( self.reader, condition=IsEqualCondition(2008), fieldNames='year').initialize().getRecords() ] self.assertEqual(len(records), 41) self.assertEqual(records[-1].getField('name').getValue(), 'Western Ogre') def test_drop_by_condition_filter_and(self): records = [ r for r in DropByConditionFilter( self.reader, condition=IsEqualCondition(2008), fieldNames=['year', 'after_year']).initialize().getRecords() ] self.assertEqual(len(records), 58) self.assertEqual(records[-1].getField('name').getValue(), 'Western Ogre') def test_drop_by_condition_filter_or(self): records = [ r for r in DropByConditionFilter( self.reader, condition=IsEqualCondition(2008), fieldNames=['year', 'after_year'], operation='OR').initialize().getRecords() ] self.assertEqual(len(records), 19) self.assertEqual(records[-1].getField('name').getValue(), 'Western Ogre') def test_keep_by_condition_filter(self): records = [ r for r in KeepByConditionFilter( self.reader, condition=IsEqualCondition(2008), fieldNames='year').initialize().getRecords() ] self.assertEqual(len(records), 44) self.assertEqual(records[-1].getField('name').getValue(), 'Sergeant Strawberry') def test_keep_by_condition_filter_and(self): records = [ r for r in KeepByConditionFilter( self.reader, condition=IsEqualCondition(2008), fieldNames=['year', 'after_year']).initialize().getRecords() ] self.assertEqual(len(records), 27) self.assertEqual(records[-1].getField('name').getValue(), 'Sergeant Strawberry') def test_keep_by_condition_filter_or(self): records = [ r for r in KeepByConditionFilter( self.reader, condition=IsEqualCondition(2008), fieldNames=['year', 'after_year'], operation='OR').initialize().getRecords() ] self.assertEqual(len(records), 66) self.assertEqual(records[-1].getField('name').getValue(), 'Sergeant Strawberry') def test_drop_field_filter(self): records = [ r for r in DropFieldFilter(self.reader, fieldNames=['email', 'after_year' ]).initialize().getRecords() ] sourceRecords = [r for r in self.reader.getRecords()] self.assertEqual(len(sourceRecords), len(records)) self.assertEqual(len(records[0].getFieldNames()), 2) self.assertEqual(records[0].getFieldNames()[-1], 'year') def test_drop_by_source_filter(self): f = DropBySourceFilter(self.reader, self.other_source, condition=IsEqualCondition('email'), join='name') f.initialize() records = [r for r in f.getRecords()] f.finalize() self.assertEqual(len(records), 2) self.assertEqual(records[-1].getField('name').getValue(), 'Nocturnal Raven')
def setUp(self): self.reader = StaticSource( FieldSet([ Field('name', StringFieldType()), Field('email', StringFieldType()), Field('year', IntegerFieldType()), Field('after_year', IntegerFieldType()), Field('number', IntegerFieldType()) ], FieldMap({ 'name': 0, 'email': 1, 'year': 2, 'after_year': 3 }))) self.reader.setResource( [['El Agent', 'El [email protected]', 2008, 2008], [ 'Serious Electron', 'Serious [email protected]', 2008, 2013 ], ['Brave Wizard', 'Brave [email protected]', 2008, 2008], [ 'Forgotten Itchy Emperor', 'Forgotten Itchy [email protected]', 2008, 2013 ], [ 'The Moving Monkey', 'The Moving [email protected]', 2008, 2008 ], [ 'Evil Ghostly Brigadier', 'Evil Ghostly [email protected]', 2008, 2013 ], [ 'Strangely Oyster', 'Strangely [email protected]', 2008, 2008 ], [ 'Anaconda Silver', 'Anaconda [email protected]', 2006, 2008 ], ['Hawk Tough', 'Hawk [email protected]', 2004, 2008], [ 'The Disappointed Craw', 'The Disappointed [email protected]', 2008, 2013 ], ['The Raven', 'The [email protected]', 1999, 2008], [ 'Ruby Boomerang', 'Ruby [email protected]', 2008, 2008 ], ['Skunk Tough', 'Skunk [email protected]', 2010, 2008], [ 'The Nervous Forgotten Major', 'The Nervous Forgotten [email protected]', 2008, 2013 ], [ 'Bursting Furious Puppet', 'Bursting Furious [email protected]', 2011, 2008 ], ['Neptune Eagle', 'Neptune [email protected]', 2011, 2013], ['The Skunk', 'The [email protected]', 2008, 2013], ['Lone Demon', 'Lone [email protected]', 2008, 2008], ['The Skunk', 'The [email protected]', 1999, 2008], [ 'Gamma Serious Spear', 'Gamma Serious [email protected]', 2008, 2008 ], [ 'Sleepy Dirty Sergeant', 'Sleepy Dirty [email protected]', 2008, 2008 ], ['Red Monkey', 'Red [email protected]', 2008, 2008], [ 'Striking Tiger', 'Striking [email protected]', 2005, 2008 ], ['Sliding Demon', 'Sliding [email protected]', 2011, 2008], [ 'Lone Commander', 'Lone [email protected]', 2008, 2013 ], ['Dragon Insane', 'Dragon [email protected]', 2013, 2013], ['Demon Skilled', 'Demon [email protected]', 2011, 2004], ['Vulture Lucky', 'Vulture [email protected]', 2003, 2008], ['The Ranger', 'The [email protected]', 2013, 2008], ['Morbid Snake', 'Morbid [email protected]', 2011, 2008], [ 'Dancing Skeleton', 'Dancing [email protected]', 2001, 2004 ], ['The Psycho', 'The [email protected]', 2005, 2008], ['Jupiter Rider', 'Jupiter [email protected]', 2011, 2008], ['Green Dog', 'Green [email protected]', 2011, 2008], [ 'Brutal Wild Colonel', 'Brutal Wild [email protected]', 2004, 2008 ], ['Random Leader', 'Random [email protected]', 2008, 2008], [ 'Pluto Brigadier', 'Pluto [email protected]', 2008, 2004 ], [ 'Southern Kangaroo', 'Southern [email protected]', 2008, 2008 ], ['Serious Flea', 'Serious [email protected]', 2001, 2005], [ 'Nocturnal Raven', 'Nocturnal [email protected]', 2008, 2004 ], ['Risky Flea', 'Risky [email protected]', 2005, 2005], ['The Corporal', 'The [email protected]', 2013, 2008], [ 'The Lucky Barbarian', 'The Lucky [email protected]', 2008, 2008 ], [ 'Rocky Serious Dog', 'Rocky Serious [email protected]', 2008, 2008 ], [ 'The Frozen Guardian', 'The Frozen [email protected]', 2008, 2008 ], [ 'Freaky Frostbite', 'Freaky [email protected]', 2008, 2004 ], [ 'The Tired Raven', 'The Tired [email protected]', 2008, 2008 ], [ 'Disappointed Frostbite', 'Disappointed [email protected]', 2008, 2008 ], ['The Craw', 'The [email protected]', 2003, 2008], [ 'Gutsy Strangely Chief', 'Gutsy Strangely [email protected]', 2008, 2008 ], ['Queen Angry', 'Queen [email protected]', 2008, 2008], [ 'Pluto Albatross', 'Pluto [email protected]', 2003, 2008 ], [ 'Endless Invader', 'Endless [email protected]', 2003, 2004 ], [ 'Beta Young Sergeant', 'Beta Young [email protected]', 2008, 2011 ], ['The Demon', 'The [email protected]', 2003, 2008], ['Lone Monkey', 'Lone [email protected]', 2011, 2008], [ 'Bursting Electron', 'Bursting [email protected]', 2003, 2010 ], [ 'Gangster Solid', 'Gangster [email protected]', 2005, 2009 ], ['The Gladiator', 'The [email protected]', 2001, 2002], [ 'Flash Frostbite', 'Flash [email protected]', 2005, 2004 ], [ 'The Rainbow Pluto Demon', 'The Rainbow Pluto [email protected]', 2011, 2013 ], [ 'Poseidon Rider', 'Poseidon [email protected]', 2008, 2006 ], [ 'The Old Alpha Brigadier', 'The Old Alpha [email protected]', 2008, 2008 ], [ 'Rough Anaconda', 'Rough [email protected]', 2001, 2011 ], [ 'Tough Dinosaur', 'Tough [email protected]', 2011, 2010 ], [ 'The Lost Dinosaur', 'The Lost [email protected]', 2008, 2008 ], ['The Raven', 'The [email protected]', 2005, 2009], ['The Agent', 'The [email protected]', 2011, 2008], [ 'Brave Scarecrow', 'Brave [email protected]', 2008, 2007 ], [ 'Flash Skeleton', 'Flash [email protected]', 2008, 2006 ], ['The Admiral', 'The [email protected]', 1998, 2005], ['The Tombstone', 'The [email protected]', 2013, 2008], ['Golden Arrow', 'Golden [email protected]', 2008, 2005], [ 'White Guardian', 'White [email protected]', 2011, 2004 ], [ 'The Black Eastern Power', 'The Black Eastern [email protected]', 2008, 2008 ], [ 'Ruthless Soldier', 'Ruthless [email protected]', 2008, 2008 ], ['Dirty Clown', 'Dirty [email protected]', 2008, 2008], ['Alpha Admiral', 'Alpha [email protected]', 2008, 2008], [ 'Lightning Major', 'Lightning [email protected]', 2008, 2008 ], [ 'The Rock Demon', 'The Rock [email protected]', 2008, 2001 ], ['Wild Tiger', 'Wild [email protected]', 2008, 2001], [ 'The Pointless Bandit', 'The Pointless [email protected]', 2008, 2008 ], ['The Sergeant', 'The [email protected]', 1998, 2002], ['Western Ogre', 'Western [email protected]', 1998, 2004], [ 'Sergeant Strawberry', 'Sergeant [email protected]', 2006, 2008 ]])
class Test_Modifier(unittest.TestCase): def setUp(self): self.reader = StaticSource( FieldSet([ Field('name', StringFieldType()), Field('email', StringFieldType()), Field('year', IntegerFieldType()), Field('after_year', IntegerFieldType()), Field('age', IntegerFieldType()) ], FieldMap({ 'name': 0, 'email': 1, 'year': 2, 'after_year': 3 }))) self.reader.setResource( [['El Agent', 'El [email protected]', 2008, 2008], [ 'Serious Electron', 'Serious [email protected]', 2008, 2013 ], ['Brave Wizard', 'Brave [email protected]', 2008, 2008], [ 'Forgotten Itchy Emperor', 'Forgotten Itchy [email protected]', 2008, 2013 ], [ 'The Moving Monkey', 'The Moving [email protected]', 2008, 2008 ], [ 'Evil Ghostly Brigadier', 'Evil Ghostly [email protected]', 2008, 2013 ], [ 'Strangely Oyster', 'Strangely [email protected]', 2008, 2008 ], [ 'Anaconda Silver', 'Anaconda [email protected]', 2006, 2008 ], ['Hawk Tough', 'Hawk [email protected]', 2004, 2008], [ 'The Disappointed Craw', 'The Disappointed [email protected]', 2008, 2013 ], ['The Raven', 'The [email protected]', 1999, 2008], [ 'Ruby Boomerang', 'Ruby [email protected]', 2008, 2008 ], ['Skunk Tough', 'Skunk [email protected]', 2010, 2008], [ 'The Nervous Forgotten Major', 'The Nervous Forgotten [email protected]', 2008, 2013 ], [ 'Bursting Furious Puppet', 'Bursting Furious [email protected]', 2011, 2008 ], ['Neptune Eagle', 'Neptune [email protected]', 2011, 2013], ['The Skunk', 'The [email protected]', 2008, 2013], ['Lone Demon', 'Lone [email protected]', 2008, 2008], ['The Skunk', 'The [email protected]', 1999, 2008], [ 'Gamma Serious Spear', 'Gamma Serious [email protected]', 2008, 2008 ], [ 'Sleepy Dirty Sergeant', 'Sleepy Dirty [email protected]', 2008, 2008 ], ['Red Monkey', 'Red [email protected]', 2008, 2008], [ 'Striking Tiger', 'Striking [email protected]', 2005, 2008 ], ['Sliding Demon', 'Sliding [email protected]', 2011, 2008], [ 'Lone Commander', 'Lone [email protected]', 2008, 2013 ], ['Dragon Insane', 'Dragon [email protected]', 2013, 2013], ['Demon Skilled', 'Demon [email protected]', 2011, 2004], ['Vulture Lucky', 'Vulture [email protected]', 2003, 2008], ['The Ranger', 'The [email protected]', 2013, 2008], ['Morbid Snake', 'Morbid [email protected]', 2011, 2008], [ 'Dancing Skeleton', 'Dancing [email protected]', 2001, 2004 ], ['The Psycho', 'The [email protected]', 2005, 2008], ['Jupiter Rider', 'Jupiter [email protected]', 2011, 2008], ['Green Dog', 'Green [email protected]', 2011, 2008], [ 'Brutal Wild Colonel', 'Brutal Wild [email protected]', 2004, 2008 ], ['Random Leader', 'Random [email protected]', 2008, 2008], [ 'Pluto Brigadier', 'Pluto [email protected]', 2008, 2004 ], [ 'Southern Kangaroo', 'Southern [email protected]', 2008, 2008 ], ['Serious Flea', 'Serious [email protected]', 2001, 2005], [ 'Nocturnal Raven', 'Nocturnal [email protected]', 2008, 2004 ], ['Risky Flea', 'Risky [email protected]', 2005, 2005], ['The Corporal', 'The [email protected]', 2013, 2008], [ 'The Lucky Barbarian', 'The Lucky [email protected]', 2008, 2008 ], [ 'Rocky Serious Dog', 'Rocky Serious [email protected]', 2008, 2008 ], [ 'The Frozen Guardian', 'The Frozen [email protected]', 2008, 2008 ], [ 'Freaky Frostbite', 'Freaky [email protected]', 2008, 2004 ], [ 'The Tired Raven', 'The Tired [email protected]', 2008, 2008 ], [ 'Disappointed Frostbite', 'Disappointed [email protected]', 2008, 2008 ], ['The Craw', 'The [email protected]', 2003, 2008], [ 'Gutsy Strangely Chief', 'Gutsy Strangely [email protected]', 2008, 2008 ], ['Queen Angry', 'Queen [email protected]', 2008, 2008], [ 'Pluto Albatross', 'Pluto [email protected]', 2003, 2008 ], [ 'Endless Invader', 'Endless [email protected]', 2003, 2004 ], [ 'Beta Young Sergeant', 'Beta Young [email protected]', 2008, 2011 ], ['The Demon', 'The [email protected]', 2003, 2008], ['Lone Monkey', 'Lone [email protected]', 2011, 2008], [ 'Bursting Electron', 'Bursting [email protected]', 2003, 2010 ], [ 'Gangster Solid', 'Gangster [email protected]', 2005, 2009 ], ['The Gladiator', 'The [email protected]', 2001, 2002], [ 'Flash Frostbite', 'Flash [email protected]', 2005, 2004 ], [ 'The Rainbow Pluto Demon', 'The Rainbow Pluto [email protected]', 2011, 2013 ], [ 'Poseidon Rider', 'Poseidon [email protected]', 2008, 2006 ], [ 'The Old Alpha Brigadier', 'The Old Alpha [email protected]', 2008, 2008 ], [ 'Rough Anaconda', 'Rough [email protected]', 2001, 2011 ], [ 'Tough Dinosaur', 'Tough [email protected]', 2011, 2010 ], [ 'The Lost Dinosaur', 'The Lost [email protected]', 2008, 2008 ], ['The Raven', 'The [email protected]', 2005, 2009], ['The Agent', 'The [email protected]', 2011, 2008], [ 'Brave Scarecrow', 'Brave [email protected]', 2008, 2007 ], [ 'Flash Skeleton', 'Flash [email protected]', 2008, 2006 ], ['The Admiral', 'The [email protected]', 1998, 2005], ['The Tombstone', 'The [email protected]', 2013, 2008], ['Golden Arrow', 'Golden [email protected]', 2008, 2005], [ 'White Guardian', 'White [email protected]', 2011, 2004 ], [ 'The Black Eastern Power', 'The Black Eastern [email protected]', 2008, 2008 ], [ 'Ruthless Soldier', 'Ruthless [email protected]', 2008, 2008 ], ['Dirty Clown', 'Dirty [email protected]', 2008, 2008], ['Alpha Admiral', 'Alpha [email protected]', 2008, 2008], [ 'Lightning Major', 'Lightning [email protected]', 2008, 2008 ], [ 'The Rock Demon', 'The Rock [email protected]', 2008, 2001 ], ['Wild Tiger', 'Wild [email protected]', 2008, 2001], [ 'The Pointless Bandit', 'The Pointless [email protected]', 2008, 2008 ], ['The Sergeant', 'The [email protected]', 1998, 2002], ['Western Ogre', 'Western [email protected]', 1998, 2004], [ 'Sergeant Strawberry', 'Sergeant [email protected]', 2006, 2008 ]]) def test_joinbykey_modifier(self): source = StaticSource( FieldSet([ Field('email', StringFieldType(), key=True), Field('age', IntegerFieldType()) ], FieldMap({ 'email': 0, 'age': 1 }))) source.setResource( [['El [email protected]', 12], ['Ochala [email protected]', 14], ['Sina [email protected]', 17], ['Akassa Savage [email protected]', 16], ['Sermak [email protected]', 22], ['Olivia Deadly [email protected]', 32], ['*****@*****.**', 42], ['Naria Cold-blodded [email protected]', 22], ['*****@*****.**', 54], ['Sina [email protected]', 56], ['Deadly [email protected]', 43], ['Mylenedriz [email protected]', 23], ['Calden [email protected]', 35], ['*****@*****.**', 56], ['Raven [email protected]', 23], ['*****@*****.**', 45], ['Pluto [email protected]', 64], ['Southern [email protected]', 53], ['Serious [email protected]', 62], ['*****@*****.**', 63], ['Risky [email protected]', 21], ['Rivatha [email protected]', 56], ['Panic [email protected]', 25], ['Tomara [email protected]', 46], ['Venessa [email protected]', 53], ['Western [email protected]', 71], ['*****@*****.**', 76]]) records = [ r for r in JoinByKeyModifier(self.reader, fieldNames=['age'], source=source).initialize().getRecords() ] self.assertEqual(len(records), 85) self.assertIsNone(records[-1].getField('age').getValue()) self.assertEqual(records[-2].getField('age').getValue(), 71) self.assertEqual(records[0].getField('age').getValue(), 12) def test_order_modifier(self): records = [ r for r in OrderModifier(self.reader, fieldNamesAndOrder=[{ 'year': 'DESC' }, { 'name': 'ASC' }]).initialize().getRecords() ] self.assertEqual(len(records), 85) self.assertEqual(records[-1].getField('year').getValue(), 1998) self.assertEqual(records[-4].getField('year').getValue(), 1999) self.assertEqual(records[0].getField('year').getValue(), 2013) self.assertEqual(records[-1].getField('name').getValue(), 'Western Ogre') self.assertEqual(records[-2].getField('name').getValue(), 'The Sergeant') self.assertEqual(records[-3].getField('name').getValue(), 'The Admiral') self.assertEqual(records[-4].getField('name').getValue(), 'The Skunk') self.assertEqual(records[0].getField('name').getValue(), 'Dragon Insane') def test_transform_field_modifier(self): records = [ r for r in TransformFieldModifier( self.reader, fieldNames=['email'], transforms=[ReplaceByRegexpTransform(regexp=u'[ ]+', to=u'') ]).initialize().getRecords() ] self.assertEqual(len(records), 85) self.assertEqual(records[-1].getField('email').getValue(), '*****@*****.**') def test_set_modifier_without_param(self): records = [ r for r in SetModifier( self.reader, fieldNames=['name']).initialize().getRecords() ] self.assertEqual(len(records), 85) self.assertEqual( len([ r for r in records if r.getField('name').getValue() is not None ]), 0) def test_set_modifier_with_sprintf_value(self): records = [ r for r in SetModifier( self.reader, fieldNames=['email'], value='%(name)s <%(email)s>').initialize().getRecords() ] self.assertEqual(len(records), 85) self.assertEqual( records[-1].getField('email').getValue(), 'Sergeant Strawberry <Sergeant [email protected]>') def test_set_modifier_value(self): records = [ r for r in SetModifier( self.reader, fieldNames=['name'], value='Same name for all').initialize().getRecords() ] self.assertEqual(len(records), 85) self.assertEqual(records[-1].getField('name').getValue(), 'Same name for all') def test_set_modifier_fn(self): def setValue(record, field, scope): if record.getField('year').getValue() == 2008: return 'Same name for 2008 years' return 'Other name for anything else' fn = setValue records = [ r for r in SetModifier( self.reader, fieldNames=['name'], fn=fn, ).initialize().getRecords() ] self.assertEqual(len(records), 85) self.assertEqual(records[-1].getField('name').getValue(), 'Other name for anything else') self.assertEqual(records[0].getField('name').getValue(), 'Same name for 2008 years') self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Same name for 2008 years' ]), 43) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Other name for anything else' ]), 42) def test_set_modifier_source(self): def setValue(record, field, scope): return 'Found same email address' \ if record.getField('email').getValue() in [ sr.getField('email').getValue() for sr in scope.getSourceRecords() ] \ else 'Not found same email address' fn = setValue source = StaticSource( FieldSet([ Field('name', StringFieldType()), Field('email', StringFieldType()) ], FieldMap({ 'name': 0, 'email': 1 }))) source.setResource( [['El Agent', 'El [email protected]'], ['Ochala Wild', 'Ochala [email protected]'], ['Sina Venomous', 'Sina [email protected]'], [ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ], ['Sermak Bad', 'Sermak [email protected]'], ['Olivia Deadly Dawod', 'Olivia Deadly [email protected]'], ['Pendus Inhuman', '*****@*****.**'], [ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ], ['Shard Brutal', '*****@*****.**'], ['Sina Cruel', 'Sina [email protected]'], ['Deadly Ohmar', 'Deadly [email protected]'], [ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ], ['Calden Frigid', 'Calden [email protected]'], ['Acid Reaper', '*****@*****.**'], ['Raven Seth', 'Raven [email protected]'], ['Random Leader', '*****@*****.**'], ['Pluto Brigadier', 'Pluto [email protected]'], ['Southern Kangaroo', 'Southern [email protected]'], ['Serious Flea', 'Serious [email protected]'], ['Nocturnal Raven', '*****@*****.**'], ['Risky Flea', 'Risky [email protected]'], ['Rivatha Todal', 'Rivatha [email protected]'], ['Panic Oliviaezit', 'Panic [email protected]'], ['Tomara Wild', 'Tomara [email protected]'], ['Venessa Metalhead', 'Venessa [email protected]'], ['Western Ogre', 'Western [email protected]'], ['Sergeant Strawberry', '*****@*****.**']]) modifier = SetModifier(self.reader, fieldNames=['name'], source=source, fn=fn) modifier.initialize() records = [r for r in modifier.getRecords()] modifier.finalize() self.assertEqual(len(records), 85) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Found same email address' ]), 6) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Not found same email address' ]), 79)