def setUp( self ):
self.reader = StaticSource(
FieldSet([
Field( 'name', StringFieldType() ),
Field( 'email', StringFieldType() )
],
FieldMap({
'name': 0,
'email': 1
}))
)
self.reader.setResource([
[ 'Ochala Wild', 'Ochala [email protected]' ],
[ 'Sina Venomous', 'Sina [email protected]' ],
[ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ],
[ 'Sermak Bad', 'Sermak [email protected]' ],
[ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ],
[ 'Pendus Inhuman', 'Pendus [email protected]' ],
[ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ],
[ 'Shard Brutal', 'Shard [email protected]' ],
[ 'Sina Cruel', 'Sina [email protected]' ],
[ 'Deadly Ohmar', 'Deadly [email protected]' ],
[ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ],
[ 'Calden Frigid', 'Calden [email protected]' ],
[ 'Acid Reaper', 'Acid [email protected]' ],
[ 'Raven Seth', 'Raven [email protected]' ],
[ 'Rivatha Todal', 'Rivatha [email protected]' ],
[ 'Panic Oliviaezit', 'Panic [email protected]' ],
[ 'Tomara Wild', 'Tomara [email protected]' ],
[ 'Venessa Metalhead', 'Venessa [email protected]' ]
])
def test_listexpander_map( self ):
static_source = StaticSource(
FieldSet([
Field( 'city', StringFieldType() ),
Field( 'geometry', ListFieldType() ),
Field( 'latitude', FloatFieldType() ),
Field( 'longitude', FloatFieldType() )
],
FieldMap({
'city': 0,
'geometry': 1
}))
)
static_source.setResource([
[ 'Balatonlelle', [[17.6874552,46.7871465],[17.6865955,46.7870049],[17.6846158,46.7866786],[17.6834977,46.7864944],[17.6822251,46.7862847],[17.6815319,46.7861705],[17.6811473,46.7861071],[17.6795989,46.785852],[17.6774482,46.7854976],[17.6739061,46.7849139],[17.6729351,46.7847539],[17.6720789,46.7846318]] ],
[ 'Balatonlelle', [[17.6871206,46.7880197],[17.6874552,46.7871465]] ],
[ 'Balatonboglar', "[[17.6709959,46.7843474],[17.6710995,46.7840514],[17.6711443,46.7838791],[17.6711725,46.7837746],[17.6713146,46.7831483]]" ]
])
expand = ListExpanderExpand( static_source, 'geometry', expanderMap = { 'latitude': 1, 'longitude': 0 } ).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 19 )
self.assertEqual( records[10].getField('city').getValue(), 'Balatonlelle' )
self.assertEqual( records[10].getField('latitude').getValue(), 46.7847539 )
self.assertEqual( records[-1].getField('city').getValue(), 'Balatonboglar' )
self.assertEqual( records[-1].getField('longitude').getValue(), 17.6713146 )
def test_listexpander( self ):
static_source = StaticSource(
FieldSet([
Field( 'namelist', ListFieldType() ),
Field( 'email', StringFieldType() ),
Field( 'year', IntegerFieldType() ),
Field( 'name', StringFieldType() )
],
FieldMap({
'namelist': 0,
'email': 1,
'year': 2
}))
)
static_source.setResource([
[ list('El Agent'), 'El [email protected]', 2008 ],
[ list('Serious Electron'), 'Serious [email protected]', 2008 ],
[ list('Brave Wizard'), 'Brave [email protected]', 2008 ]
])
expand = ListExpanderExpand( static_source, 'namelist', 'name' ).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), len(list('El Agent')) + len(list('Serious Electron')) + len(list('Brave Wizard')) )
self.assertEqual( records[-1].getField('name').getValue(), 'd' )
self.assertEqual( records[-1].getField('email').getValue(), 'Brave [email protected]' )
def test_joinbykey_modifier( self ):
source = StaticSource(
FieldSet([
Field( 'email', StringFieldType(), key = True ),
Field( 'age', IntegerFieldType() )
],
FieldMap({
'email': 0,
'age': 1
}))
)
source.setResource([
[ 'El [email protected]', 12 ],
[ 'Ochala [email protected]', 14 ],
[ 'Sina [email protected]', 17 ],
[ 'Akassa Savage [email protected]', 16 ],
[ 'Sermak [email protected]', 22 ],
[ 'Olivia Deadly [email protected]', 32 ],
[ '*****@*****.**', 42 ],
[ 'Naria Cold-blodded [email protected]', 22 ],
[ '*****@*****.**', 54 ],
[ 'Sina [email protected]', 56 ],
[ 'Deadly [email protected]', 43 ],
[ 'Mylenedriz [email protected]', 23 ],
[ 'Calden [email protected]', 35 ],
[ '*****@*****.**', 56 ],
[ 'Raven [email protected]', 23 ],
[ '*****@*****.**', 45 ],
[ 'Pluto [email protected]', 64 ],
[ 'Southern [email protected]', 53 ],
[ 'Serious [email protected]', 62 ],
[ '*****@*****.**', 63 ],
[ 'Risky [email protected]', 21 ],
[ 'Rivatha [email protected]', 56 ],
[ 'Panic [email protected]', 25 ],
[ 'Tomara [email protected]', 46 ],
[ 'Venessa [email protected]', 53 ],
[ 'Western [email protected]', 71 ],
[ '*****@*****.**', 76 ]
])
records = [ r for r in JoinByKeyModifier(
self.reader,
fieldNames = [ 'age' ],
source = source
).initialize().getRecords() ]
self.assertEqual( len( records ), 85 )
self.assertIsNone( records[-1].getField('age').getValue() )
self.assertEqual( records[-2].getField('age').getValue(), 71 )
self.assertEqual( records[0].getField('age').getValue(), 12 )
def test_joinbykey_modifier(self):
source = StaticSource(
FieldSet([
Field('email', StringFieldType(), key=True),
Field('age', IntegerFieldType())
], FieldMap({
'email': 0,
'age': 1
})))
source.setResource(
[['El [email protected]', 12],
['Ochala [email protected]', 14],
['Sina [email protected]', 17],
['Akassa Savage [email protected]', 16],
['Sermak [email protected]', 22],
['Olivia Deadly [email protected]', 32],
['*****@*****.**', 42],
['Naria Cold-blodded [email protected]', 22],
['*****@*****.**', 54],
['Sina [email protected]', 56],
['Deadly [email protected]', 43],
['Mylenedriz [email protected]', 23],
['Calden [email protected]', 35],
['*****@*****.**', 56],
['Raven [email protected]', 23],
['*****@*****.**', 45],
['Pluto [email protected]', 64],
['Southern [email protected]', 53],
['Serious [email protected]', 62],
['*****@*****.**', 63],
['Risky [email protected]', 21],
['Rivatha [email protected]', 56],
['Panic [email protected]', 25],
['Tomara [email protected]', 46],
['Venessa [email protected]', 53],
['Western [email protected]', 71],
['*****@*****.**', 76]])
records = [
r for r in
JoinByKeyModifier(self.reader, fieldNames=['age'],
source=source).initialize().getRecords()
]
self.assertEqual(len(records), 85)
self.assertIsNone(records[-1].getField('age').getValue())
self.assertEqual(records[-2].getField('age').getValue(), 71)
self.assertEqual(records[0].getField('age').getValue(), 12)
def test_field( self ):
static_source = StaticSource(
FieldSet([
Field( 'year', IntegerFieldType() ),
Field( 'country', StringFieldType() ),
Field( 'count', IntegerFieldType() ),
Field( 'cz', IntegerFieldType() ),
Field( 'hu', IntegerFieldType() ),
Field( 'sk', IntegerFieldType() ),
Field( 'pl', IntegerFieldType() )
],
FieldMap({
'year': 0,
'cz': 1,
'hu': 2,
'sk': 3,
'pl': 4
}))
)
static_source.setResource([
[1999,32,694,129,230],
[1999,395,392,297,453],
[1999,635,812,115,97]
])
expand = FieldExpand(
static_source,
fieldNamesAndLabels = {
'cz': 'Czech',
'hu': 'Hungary',
'sk': 'Slovak',
'pl': 'Poland',
},
valueFieldName = 'count',
labelFieldName = 'country'
).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 12 )
self.assertEqual( records[-1].getField('year').getValue(), 1999 )
self.assertEqual( records[-1].getField('count').getValue(), 812 )
self.assertEqual( records[-1].getField('pl').getValue(), 97 )
self.assertEqual( records[-1].getField('country').getValue(), 'Hungary' )
def test_melt( self ):
static_source = StaticSource(
FieldSet([
Field( 'first', StringFieldType() ),
Field( 'height', FloatFieldType() ),
Field( 'last', StringFieldType() ),
Field( 'weight', IntegerFieldType() ),
Field( 'iq', IntegerFieldType() ),
Field( 'quantity', StringFieldType() ),
Field( 'value', FloatFieldType() )
],
FieldMap({
'first': 0,
'height': 1,
'last': 2,
'weight': 3,
'iq': 4
}))
)
static_source.setResource([
['John',5.5,'Doe',130,102],
['Mary',6.0,'Bo',150,98]
])
expand = MeltExpand(
static_source,
fieldNames = ['first','last'],
valueFieldName = 'value',
labelFieldName = 'quantity'
).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 6 )
self.assertEqual( len( records[0].getFieldNames() ), 4)
self.assertEqual( records[-1].getField('first').getValue(), 'Mary' )
self.assertEqual( records[-1].getField('last').getValue(), 'Bo' )
self.assertEqual( records[-1].getField('quantity').getValue(), 'height' )
self.assertEqual( records[-1].getField('value').getValue(), 6.0 )
def setUp( self ):
self.reader = StaticSource(
FieldSet([
Field( 'name', StringFieldType() ),
Field( 'email', StringFieldType() ),
Field( 'year', IntegerFieldType() ),
Field( 'after_year', IntegerFieldType() ),
Field( 'age', IntegerFieldType() )
],
FieldMap({
'name': 0,
'email': 1,
'year': 2,
'after_year': 3
}))
)
self.reader.setResource([
[ 'El Agent', 'El [email protected]', 2008, 2008 ],
[ 'Serious Electron', 'Serious [email protected]', 2008, 2013 ],
[ 'Brave Wizard', 'Brave [email protected]', 2008, 2008 ],
[ 'Forgotten Itchy Emperor', 'Forgotten Itchy [email protected]', 2008, 2013 ],
[ 'The Moving Monkey', 'The Moving [email protected]', 2008, 2008 ],
[ 'Evil Ghostly Brigadier', 'Evil Ghostly [email protected]', 2008, 2013 ],
[ 'Strangely Oyster', 'Strangely [email protected]', 2008, 2008 ],
[ 'Anaconda Silver', 'Anaconda [email protected]', 2006, 2008 ],
[ 'Hawk Tough', 'Hawk [email protected]', 2004, 2008 ],
[ 'The Disappointed Craw', 'The Disappointed [email protected]', 2008, 2013 ],
[ 'The Raven', 'The [email protected]', 1999, 2008 ],
[ 'Ruby Boomerang', 'Ruby [email protected]', 2008, 2008 ],
[ 'Skunk Tough', 'Skunk [email protected]', 2010, 2008 ],
[ 'The Nervous Forgotten Major', 'The Nervous Forgotten [email protected]', 2008, 2013 ],
[ 'Bursting Furious Puppet', 'Bursting Furious [email protected]', 2011, 2008 ],
[ 'Neptune Eagle', 'Neptune [email protected]', 2011, 2013 ],
[ 'The Skunk', 'The [email protected]', 2008, 2013 ],
[ 'Lone Demon', 'Lone [email protected]', 2008, 2008 ],
[ 'The Skunk', 'The [email protected]', 1999, 2008 ],
[ 'Gamma Serious Spear', 'Gamma Serious [email protected]', 2008, 2008 ],
[ 'Sleepy Dirty Sergeant', 'Sleepy Dirty [email protected]', 2008, 2008 ],
[ 'Red Monkey', 'Red [email protected]', 2008, 2008 ],
[ 'Striking Tiger', 'Striking [email protected]', 2005, 2008 ],
[ 'Sliding Demon', 'Sliding [email protected]', 2011, 2008 ],
[ 'Lone Commander', 'Lone [email protected]', 2008, 2013 ],
[ 'Dragon Insane', 'Dragon [email protected]', 2013, 2013 ],
[ 'Demon Skilled', 'Demon [email protected]', 2011, 2004 ],
[ 'Vulture Lucky', 'Vulture [email protected]', 2003, 2008 ],
[ 'The Ranger', 'The [email protected]', 2013, 2008 ],
[ 'Morbid Snake', 'Morbid [email protected]', 2011, 2008 ],
[ 'Dancing Skeleton', 'Dancing [email protected]', 2001, 2004 ],
[ 'The Psycho', 'The [email protected]', 2005, 2008 ],
[ 'Jupiter Rider', 'Jupiter [email protected]', 2011, 2008 ],
[ 'Green Dog', 'Green [email protected]', 2011, 2008 ],
[ 'Brutal Wild Colonel', 'Brutal Wild [email protected]', 2004, 2008 ],
[ 'Random Leader', 'Random [email protected]', 2008, 2008 ],
[ 'Pluto Brigadier', 'Pluto [email protected]', 2008, 2004 ],
[ 'Southern Kangaroo', 'Southern [email protected]', 2008, 2008 ],
[ 'Serious Flea', 'Serious [email protected]', 2001, 2005 ],
[ 'Nocturnal Raven', 'Nocturnal [email protected]', 2008, 2004 ],
[ 'Risky Flea', 'Risky [email protected]', 2005, 2005 ],
[ 'The Corporal', 'The [email protected]', 2013, 2008 ],
[ 'The Lucky Barbarian', 'The Lucky [email protected]', 2008, 2008 ],
[ 'Rocky Serious Dog', 'Rocky Serious [email protected]', 2008, 2008 ],
[ 'The Frozen Guardian', 'The Frozen [email protected]', 2008, 2008 ],
[ 'Freaky Frostbite', 'Freaky [email protected]', 2008, 2004 ],
[ 'The Tired Raven', 'The Tired [email protected]', 2008, 2008 ],
[ 'Disappointed Frostbite', 'Disappointed [email protected]', 2008, 2008 ],
[ 'The Craw', 'The [email protected]', 2003, 2008 ],
[ 'Gutsy Strangely Chief', 'Gutsy Strangely [email protected]', 2008, 2008 ],
[ 'Queen Angry', 'Queen [email protected]', 2008, 2008 ],
[ 'Pluto Albatross', 'Pluto [email protected]', 2003, 2008 ],
[ 'Endless Invader', 'Endless [email protected]', 2003, 2004 ],
[ 'Beta Young Sergeant', 'Beta Young [email protected]', 2008, 2011 ],
[ 'The Demon', 'The [email protected]', 2003, 2008 ],
[ 'Lone Monkey', 'Lone [email protected]', 2011, 2008 ],
[ 'Bursting Electron', 'Bursting [email protected]', 2003, 2010 ],
[ 'Gangster Solid', 'Gangster [email protected]', 2005, 2009 ],
[ 'The Gladiator', 'The [email protected]', 2001, 2002 ],
[ 'Flash Frostbite', 'Flash [email protected]', 2005, 2004 ],
[ 'The Rainbow Pluto Demon', 'The Rainbow Pluto [email protected]', 2011, 2013 ],
[ 'Poseidon Rider', 'Poseidon [email protected]', 2008, 2006 ],
[ 'The Old Alpha Brigadier', 'The Old Alpha [email protected]', 2008, 2008 ],
[ 'Rough Anaconda', 'Rough [email protected]', 2001, 2011 ],
[ 'Tough Dinosaur', 'Tough [email protected]', 2011, 2010 ],
[ 'The Lost Dinosaur', 'The Lost [email protected]', 2008, 2008 ],
[ 'The Raven', 'The [email protected]', 2005, 2009 ],
[ 'The Agent', 'The [email protected]', 2011, 2008 ],
[ 'Brave Scarecrow', 'Brave [email protected]', 2008, 2007 ],
[ 'Flash Skeleton', 'Flash [email protected]', 2008, 2006 ],
[ 'The Admiral', 'The [email protected]', 1998, 2005 ],
[ 'The Tombstone', 'The [email protected]', 2013, 2008 ],
[ 'Golden Arrow', 'Golden [email protected]', 2008, 2005 ],
[ 'White Guardian', 'White [email protected]', 2011, 2004 ],
[ 'The Black Eastern Power', 'The Black Eastern [email protected]', 2008, 2008 ],
[ 'Ruthless Soldier', 'Ruthless [email protected]', 2008, 2008 ],
[ 'Dirty Clown', 'Dirty [email protected]', 2008, 2008 ],
[ 'Alpha Admiral', 'Alpha [email protected]', 2008, 2008 ],
[ 'Lightning Major', 'Lightning [email protected]', 2008, 2008 ],
[ 'The Rock Demon', 'The Rock [email protected]', 2008, 2001 ],
[ 'Wild Tiger', 'Wild [email protected]', 2008, 2001 ],
[ 'The Pointless Bandit', 'The Pointless [email protected]', 2008, 2008 ],
[ 'The Sergeant', 'The [email protected]', 1998, 2002 ],
[ 'Western Ogre', 'Western [email protected]', 1998, 2004 ],
[ 'Sergeant Strawberry', 'Sergeant [email protected]', 2006, 2008 ]
])
class Test_Modifier( unittest.TestCase ):
def setUp( self ):
self.reader = StaticSource(
FieldSet([
Field( 'name', StringFieldType() ),
Field( 'email', StringFieldType() ),
Field( 'year', IntegerFieldType() ),
Field( 'after_year', IntegerFieldType() ),
Field( 'age', IntegerFieldType() )
],
FieldMap({
'name': 0,
'email': 1,
'year': 2,
'after_year': 3
}))
)
self.reader.setResource([
[ 'El Agent', 'El [email protected]', 2008, 2008 ],
[ 'Serious Electron', 'Serious [email protected]', 2008, 2013 ],
[ 'Brave Wizard', 'Brave [email protected]', 2008, 2008 ],
[ 'Forgotten Itchy Emperor', 'Forgotten Itchy [email protected]', 2008, 2013 ],
[ 'The Moving Monkey', 'The Moving [email protected]', 2008, 2008 ],
[ 'Evil Ghostly Brigadier', 'Evil Ghostly [email protected]', 2008, 2013 ],
[ 'Strangely Oyster', 'Strangely [email protected]', 2008, 2008 ],
[ 'Anaconda Silver', 'Anaconda [email protected]', 2006, 2008 ],
[ 'Hawk Tough', 'Hawk [email protected]', 2004, 2008 ],
[ 'The Disappointed Craw', 'The Disappointed [email protected]', 2008, 2013 ],
[ 'The Raven', 'The [email protected]', 1999, 2008 ],
[ 'Ruby Boomerang', 'Ruby [email protected]', 2008, 2008 ],
[ 'Skunk Tough', 'Skunk [email protected]', 2010, 2008 ],
[ 'The Nervous Forgotten Major', 'The Nervous Forgotten [email protected]', 2008, 2013 ],
[ 'Bursting Furious Puppet', 'Bursting Furious [email protected]', 2011, 2008 ],
[ 'Neptune Eagle', 'Neptune [email protected]', 2011, 2013 ],
[ 'The Skunk', 'The [email protected]', 2008, 2013 ],
[ 'Lone Demon', 'Lone [email protected]', 2008, 2008 ],
[ 'The Skunk', 'The [email protected]', 1999, 2008 ],
[ 'Gamma Serious Spear', 'Gamma Serious [email protected]', 2008, 2008 ],
[ 'Sleepy Dirty Sergeant', 'Sleepy Dirty [email protected]', 2008, 2008 ],
[ 'Red Monkey', 'Red [email protected]', 2008, 2008 ],
[ 'Striking Tiger', 'Striking [email protected]', 2005, 2008 ],
[ 'Sliding Demon', 'Sliding [email protected]', 2011, 2008 ],
[ 'Lone Commander', 'Lone [email protected]', 2008, 2013 ],
[ 'Dragon Insane', 'Dragon [email protected]', 2013, 2013 ],
[ 'Demon Skilled', 'Demon [email protected]', 2011, 2004 ],
[ 'Vulture Lucky', 'Vulture [email protected]', 2003, 2008 ],
[ 'The Ranger', 'The [email protected]', 2013, 2008 ],
[ 'Morbid Snake', 'Morbid [email protected]', 2011, 2008 ],
[ 'Dancing Skeleton', 'Dancing [email protected]', 2001, 2004 ],
[ 'The Psycho', 'The [email protected]', 2005, 2008 ],
[ 'Jupiter Rider', 'Jupiter [email protected]', 2011, 2008 ],
[ 'Green Dog', 'Green [email protected]', 2011, 2008 ],
[ 'Brutal Wild Colonel', 'Brutal Wild [email protected]', 2004, 2008 ],
[ 'Random Leader', 'Random [email protected]', 2008, 2008 ],
[ 'Pluto Brigadier', 'Pluto [email protected]', 2008, 2004 ],
[ 'Southern Kangaroo', 'Southern [email protected]', 2008, 2008 ],
[ 'Serious Flea', 'Serious [email protected]', 2001, 2005 ],
[ 'Nocturnal Raven', 'Nocturnal [email protected]', 2008, 2004 ],
[ 'Risky Flea', 'Risky [email protected]', 2005, 2005 ],
[ 'The Corporal', 'The [email protected]', 2013, 2008 ],
[ 'The Lucky Barbarian', 'The Lucky [email protected]', 2008, 2008 ],
[ 'Rocky Serious Dog', 'Rocky Serious [email protected]', 2008, 2008 ],
[ 'The Frozen Guardian', 'The Frozen [email protected]', 2008, 2008 ],
[ 'Freaky Frostbite', 'Freaky [email protected]', 2008, 2004 ],
[ 'The Tired Raven', 'The Tired [email protected]', 2008, 2008 ],
[ 'Disappointed Frostbite', 'Disappointed [email protected]', 2008, 2008 ],
[ 'The Craw', 'The [email protected]', 2003, 2008 ],
[ 'Gutsy Strangely Chief', 'Gutsy Strangely [email protected]', 2008, 2008 ],
[ 'Queen Angry', 'Queen [email protected]', 2008, 2008 ],
[ 'Pluto Albatross', 'Pluto [email protected]', 2003, 2008 ],
[ 'Endless Invader', 'Endless [email protected]', 2003, 2004 ],
[ 'Beta Young Sergeant', 'Beta Young [email protected]', 2008, 2011 ],
[ 'The Demon', 'The [email protected]', 2003, 2008 ],
[ 'Lone Monkey', 'Lone [email protected]', 2011, 2008 ],
[ 'Bursting Electron', 'Bursting [email protected]', 2003, 2010 ],
[ 'Gangster Solid', 'Gangster [email protected]', 2005, 2009 ],
[ 'The Gladiator', 'The [email protected]', 2001, 2002 ],
[ 'Flash Frostbite', 'Flash [email protected]', 2005, 2004 ],
[ 'The Rainbow Pluto Demon', 'The Rainbow Pluto [email protected]', 2011, 2013 ],
[ 'Poseidon Rider', 'Poseidon [email protected]', 2008, 2006 ],
[ 'The Old Alpha Brigadier', 'The Old Alpha [email protected]', 2008, 2008 ],
[ 'Rough Anaconda', 'Rough [email protected]', 2001, 2011 ],
[ 'Tough Dinosaur', 'Tough [email protected]', 2011, 2010 ],
[ 'The Lost Dinosaur', 'The Lost [email protected]', 2008, 2008 ],
[ 'The Raven', 'The [email protected]', 2005, 2009 ],
[ 'The Agent', 'The [email protected]', 2011, 2008 ],
[ 'Brave Scarecrow', 'Brave [email protected]', 2008, 2007 ],
[ 'Flash Skeleton', 'Flash [email protected]', 2008, 2006 ],
[ 'The Admiral', 'The [email protected]', 1998, 2005 ],
[ 'The Tombstone', 'The [email protected]', 2013, 2008 ],
[ 'Golden Arrow', 'Golden [email protected]', 2008, 2005 ],
[ 'White Guardian', 'White [email protected]', 2011, 2004 ],
[ 'The Black Eastern Power', 'The Black Eastern [email protected]', 2008, 2008 ],
[ 'Ruthless Soldier', 'Ruthless [email protected]', 2008, 2008 ],
[ 'Dirty Clown', 'Dirty [email protected]', 2008, 2008 ],
[ 'Alpha Admiral', 'Alpha [email protected]', 2008, 2008 ],
[ 'Lightning Major', 'Lightning [email protected]', 2008, 2008 ],
[ 'The Rock Demon', 'The Rock [email protected]', 2008, 2001 ],
[ 'Wild Tiger', 'Wild [email protected]', 2008, 2001 ],
[ 'The Pointless Bandit', 'The Pointless [email protected]', 2008, 2008 ],
[ 'The Sergeant', 'The [email protected]', 1998, 2002 ],
[ 'Western Ogre', 'Western [email protected]', 1998, 2004 ],
[ 'Sergeant Strawberry', 'Sergeant [email protected]', 2006, 2008 ]
])
def test_joinbykey_modifier( self ):
source = StaticSource(
FieldSet([
Field( 'email', StringFieldType(), key = True ),
Field( 'age', IntegerFieldType() )
],
FieldMap({
'email': 0,
'age': 1
}))
)
source.setResource([
[ 'El [email protected]', 12 ],
[ 'Ochala [email protected]', 14 ],
[ 'Sina [email protected]', 17 ],
[ 'Akassa Savage [email protected]', 16 ],
[ 'Sermak [email protected]', 22 ],
[ 'Olivia Deadly [email protected]', 32 ],
[ '*****@*****.**', 42 ],
[ 'Naria Cold-blodded [email protected]', 22 ],
[ '*****@*****.**', 54 ],
[ 'Sina [email protected]', 56 ],
[ 'Deadly [email protected]', 43 ],
[ 'Mylenedriz [email protected]', 23 ],
[ 'Calden [email protected]', 35 ],
[ '*****@*****.**', 56 ],
[ 'Raven [email protected]', 23 ],
[ '*****@*****.**', 45 ],
[ 'Pluto [email protected]', 64 ],
[ 'Southern [email protected]', 53 ],
[ 'Serious [email protected]', 62 ],
[ '*****@*****.**', 63 ],
[ 'Risky [email protected]', 21 ],
[ 'Rivatha [email protected]', 56 ],
[ 'Panic [email protected]', 25 ],
[ 'Tomara [email protected]', 46 ],
[ 'Venessa [email protected]', 53 ],
[ 'Western [email protected]', 71 ],
[ '*****@*****.**', 76 ]
])
records = [ r for r in JoinByKeyModifier(
self.reader,
fieldNames = [ 'age' ],
source = source
).initialize().getRecords() ]
self.assertEqual( len( records ), 85 )
self.assertIsNone( records[-1].getField('age').getValue() )
self.assertEqual( records[-2].getField('age').getValue(), 71 )
self.assertEqual( records[0].getField('age').getValue(), 12 )
def test_order_modifier( self ):
records = [ r for r in OrderModifier(
self.reader,
fieldNamesAndOrder = [ { 'year': 'DESC' }, { 'name': 'ASC' } ]
).initialize().getRecords() ]
self.assertEqual( len( records ), 85 )
self.assertEqual( records[-1].getField('year').getValue(), 1998 )
self.assertEqual( records[-4].getField('year').getValue(), 1999 )
self.assertEqual( records[0].getField('year').getValue(), 2013 )
self.assertEqual( records[-1].getField('name').getValue(), 'Western Ogre' )
self.assertEqual( records[-2].getField('name').getValue(), 'The Sergeant' )
self.assertEqual( records[-3].getField('name').getValue(), 'The Admiral' )
self.assertEqual( records[-4].getField('name').getValue(), 'The Skunk' )
self.assertEqual( records[0].getField('name').getValue(), 'Dragon Insane' )
def test_transform_field_modifier( self ):
records = [ r for r in TransformFieldModifier(
self.reader,
fieldNames = [ 'email' ],
transforms = [ ReplaceByRegexpTransform( regexp = u'[ ]+', to = u'' ) ]
).initialize().getRecords() ]
self.assertEqual( len( records ), 85 )
self.assertEqual( records[-1].getField('email').getValue(), '*****@*****.**' )
def test_set_modifier_without_param( self ):
records = [ r for r in SetModifier(
self.reader,
fieldNames = [ 'name' ]
).initialize().getRecords() ]
self.assertEqual( len( records ), 85 )
self.assertEqual( len([ r for r in records if r.getField('name').getValue() is not None ]), 0 )
def test_set_modifier_with_sprintf_value( self ):
records = [ r for r in SetModifier(
self.reader,
fieldNames = [ 'email' ],
value = '%(name)s <%(email)s>'
).initialize().getRecords() ]
self.assertEqual( len( records ), 85 )
self.assertEqual( records[-1].getField('email').getValue(), 'Sergeant Strawberry <Sergeant [email protected]>' )
def test_set_modifier_value( self ):
records = [ r for r in SetModifier(
self.reader,
fieldNames = [ 'name' ],
value = 'Same name for all'
).initialize().getRecords() ]
self.assertEqual( len( records ), 85 )
self.assertEqual( records[-1].getField('name').getValue(), 'Same name for all' )
def test_set_modifier_fn( self ):
def setValue( record, field, scope ):
if record.getField('year').getValue() == 2008:
return 'Same name for 2008 years'
return 'Other name for anything else'
fn = setValue
records = [ r for r in SetModifier(
self.reader,
fieldNames = [ 'name' ],
fn = fn,
).initialize().getRecords() ]
self.assertEqual( len( records ), 85 )
self.assertEqual( records[-1].getField('name').getValue(), 'Other name for anything else' )
self.assertEqual( records[0].getField('name').getValue(), 'Same name for 2008 years' )
self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Same name for 2008 years' ]), 43 )
self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Other name for anything else' ]), 42 )
def test_set_modifier_source( self ):
def setValue( record, field, scope ):
return 'Found same email address' \
if record.getField('email').getValue() in [ sr.getField('email').getValue() for sr in scope.getSourceRecords() ] \
else 'Not found same email address'
fn = setValue
source = StaticSource(
FieldSet([
Field( 'name', StringFieldType() ),
Field( 'email', StringFieldType() )
],
FieldMap({
'name': 0,
'email': 1
}))
)
source.setResource([
[ 'El Agent', 'El [email protected]' ],
[ 'Ochala Wild', 'Ochala [email protected]' ],
[ 'Sina Venomous', 'Sina [email protected]' ],
[ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ],
[ 'Sermak Bad', 'Sermak [email protected]' ],
[ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ],
[ 'Pendus Inhuman', '*****@*****.**' ],
[ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ],
[ 'Shard Brutal', '*****@*****.**' ],
[ 'Sina Cruel', 'Sina [email protected]' ],
[ 'Deadly Ohmar', 'Deadly [email protected]' ],
[ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ],
[ 'Calden Frigid', 'Calden [email protected]' ],
[ 'Acid Reaper', '*****@*****.**' ],
[ 'Raven Seth', 'Raven [email protected]' ],
[ 'Random Leader', '*****@*****.**' ],
[ 'Pluto Brigadier', 'Pluto [email protected]' ],
[ 'Southern Kangaroo', 'Southern [email protected]' ],
[ 'Serious Flea', 'Serious [email protected]' ],
[ 'Nocturnal Raven', '*****@*****.**' ],
[ 'Risky Flea', 'Risky [email protected]' ],
[ 'Rivatha Todal', 'Rivatha [email protected]' ],
[ 'Panic Oliviaezit', 'Panic [email protected]' ],
[ 'Tomara Wild', 'Tomara [email protected]' ],
[ 'Venessa Metalhead', 'Venessa [email protected]' ],
[ 'Western Ogre', 'Western [email protected]' ],
[ 'Sergeant Strawberry', '*****@*****.**' ]
])
modifier = SetModifier(
self.reader,
fieldNames = [ 'name' ],
source = source,
fn = fn
)
modifier.initialize()
records = [ r for r in modifier.getRecords() ]
modifier.finalize()
self.assertEqual( len( records ), 85 )
self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Found same email address' ]), 6 )
self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Not found same email address' ]), 79 )
def test_set_modifier_source( self ):
def setValue( record, field, scope ):
return 'Found same email address' \
if record.getField('email').getValue() in [ sr.getField('email').getValue() for sr in scope.getSourceRecords() ] \
else 'Not found same email address'
fn = setValue
source = StaticSource(
FieldSet([
Field( 'name', StringFieldType() ),
Field( 'email', StringFieldType() )
],
FieldMap({
'name': 0,
'email': 1
}))
)
source.setResource([
[ 'El Agent', 'El [email protected]' ],
[ 'Ochala Wild', 'Ochala [email protected]' ],
[ 'Sina Venomous', 'Sina [email protected]' ],
[ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ],
[ 'Sermak Bad', 'Sermak [email protected]' ],
[ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ],
[ 'Pendus Inhuman', '*****@*****.**' ],
[ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ],
[ 'Shard Brutal', '*****@*****.**' ],
[ 'Sina Cruel', 'Sina [email protected]' ],
[ 'Deadly Ohmar', 'Deadly [email protected]' ],
[ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ],
[ 'Calden Frigid', 'Calden [email protected]' ],
[ 'Acid Reaper', '*****@*****.**' ],
[ 'Raven Seth', 'Raven [email protected]' ],
[ 'Random Leader', '*****@*****.**' ],
[ 'Pluto Brigadier', 'Pluto [email protected]' ],
[ 'Southern Kangaroo', 'Southern [email protected]' ],
[ 'Serious Flea', 'Serious [email protected]' ],
[ 'Nocturnal Raven', '*****@*****.**' ],
[ 'Risky Flea', 'Risky [email protected]' ],
[ 'Rivatha Todal', 'Rivatha [email protected]' ],
[ 'Panic Oliviaezit', 'Panic [email protected]' ],
[ 'Tomara Wild', 'Tomara [email protected]' ],
[ 'Venessa Metalhead', 'Venessa [email protected]' ],
[ 'Western Ogre', 'Western [email protected]' ],
[ 'Sergeant Strawberry', '*****@*****.**' ]
])
modifier = SetModifier(
self.reader,
fieldNames = [ 'name' ],
source = source,
fn = fn
)
modifier.initialize()
records = [ r for r in modifier.getRecords() ]
modifier.finalize()
self.assertEqual( len( records ), 85 )
self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Found same email address' ]), 6 )
self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Not found same email address' ]), 79 )
def setUp(self):
self.reader = StaticSource(
FieldSet(
[
Field("name", StringFieldType()),
Field("email", StringFieldType()),
Field("year", IntegerFieldType()),
Field("after_year", IntegerFieldType()),
Field("number", IntegerFieldType()),
],
FieldMap({"name": 0, "email": 1, "year": 2, "after_year": 3}),
)
)
self.reader.setResource(
[
["El Agent", "El [email protected]", 2008, 2008],
["Serious Electron", "Serious [email protected]", 2008, 2013],
["Brave Wizard", "Brave [email protected]", 2008, 2008],
["Forgotten Itchy Emperor", "Forgotten Itchy [email protected]", 2008, 2013],
["The Moving Monkey", "The Moving [email protected]", 2008, 2008],
["Evil Ghostly Brigadier", "Evil Ghostly [email protected]", 2008, 2013],
["Strangely Oyster", "Strangely [email protected]", 2008, 2008],
["Anaconda Silver", "Anaconda [email protected]", 2006, 2008],
["Hawk Tough", "Hawk [email protected]", 2004, 2008],
["The Disappointed Craw", "The Disappointed [email protected]", 2008, 2013],
["The Raven", "The [email protected]", 1999, 2008],
["Ruby Boomerang", "Ruby [email protected]", 2008, 2008],
["Skunk Tough", "Skunk [email protected]", 2010, 2008],
["The Nervous Forgotten Major", "The Nervous Forgotten [email protected]", 2008, 2013],
["Bursting Furious Puppet", "Bursting Furious [email protected]", 2011, 2008],
["Neptune Eagle", "Neptune [email protected]", 2011, 2013],
["The Skunk", "The [email protected]", 2008, 2013],
["Lone Demon", "Lone [email protected]", 2008, 2008],
["The Skunk", "The [email protected]", 1999, 2008],
["Gamma Serious Spear", "Gamma Serious [email protected]", 2008, 2008],
["Sleepy Dirty Sergeant", "Sleepy Dirty [email protected]", 2008, 2008],
["Red Monkey", "Red [email protected]", 2008, 2008],
["Striking Tiger", "Striking [email protected]", 2005, 2008],
["Sliding Demon", "Sliding [email protected]", 2011, 2008],
["Lone Commander", "Lone [email protected]", 2008, 2013],
["Dragon Insane", "Dragon [email protected]", 2013, 2013],
["Demon Skilled", "Demon [email protected]", 2011, 2004],
["Vulture Lucky", "Vulture [email protected]", 2003, 2008],
["The Ranger", "The [email protected]", 2013, 2008],
["Morbid Snake", "Morbid [email protected]", 2011, 2008],
["Dancing Skeleton", "Dancing [email protected]", 2001, 2004],
["The Psycho", "The [email protected]", 2005, 2008],
["Jupiter Rider", "Jupiter [email protected]", 2011, 2008],
["Green Dog", "Green [email protected]", 2011, 2008],
["Brutal Wild Colonel", "Brutal Wild [email protected]", 2004, 2008],
["Random Leader", "Random [email protected]", 2008, 2008],
["Pluto Brigadier", "Pluto [email protected]", 2008, 2004],
["Southern Kangaroo", "Southern [email protected]", 2008, 2008],
["Serious Flea", "Serious [email protected]", 2001, 2005],
["Nocturnal Raven", "Nocturnal [email protected]", 2008, 2004],
["Risky Flea", "Risky [email protected]", 2005, 2005],
["The Corporal", "The [email protected]", 2013, 2008],
["The Lucky Barbarian", "The Lucky [email protected]", 2008, 2008],
["Rocky Serious Dog", "Rocky Serious [email protected]", 2008, 2008],
["The Frozen Guardian", "The Frozen [email protected]", 2008, 2008],
["Freaky Frostbite", "Freaky [email protected]", 2008, 2004],
["The Tired Raven", "The Tired [email protected]", 2008, 2008],
["Disappointed Frostbite", "Disappointed [email protected]", 2008, 2008],
["The Craw", "The [email protected]", 2003, 2008],
["Gutsy Strangely Chief", "Gutsy Strangely [email protected]", 2008, 2008],
["Queen Angry", "Queen [email protected]", 2008, 2008],
["Pluto Albatross", "Pluto [email protected]", 2003, 2008],
["Endless Invader", "Endless [email protected]", 2003, 2004],
["Beta Young Sergeant", "Beta Young [email protected]", 2008, 2011],
["The Demon", "The [email protected]", 2003, 2008],
["Lone Monkey", "Lone [email protected]", 2011, 2008],
["Bursting Electron", "Bursting [email protected]", 2003, 2010],
["Gangster Solid", "Gangster [email protected]", 2005, 2009],
["The Gladiator", "The [email protected]", 2001, 2002],
["Flash Frostbite", "Flash [email protected]", 2005, 2004],
["The Rainbow Pluto Demon", "The Rainbow Pluto [email protected]", 2011, 2013],
["Poseidon Rider", "Poseidon [email protected]", 2008, 2006],
["The Old Alpha Brigadier", "The Old Alpha [email protected]", 2008, 2008],
["Rough Anaconda", "Rough [email protected]", 2001, 2011],
["Tough Dinosaur", "Tough [email protected]", 2011, 2010],
["The Lost Dinosaur", "The Lost [email protected]", 2008, 2008],
["The Raven", "The [email protected]", 2005, 2009],
["The Agent", "The [email protected]", 2011, 2008],
["Brave Scarecrow", "Brave [email protected]", 2008, 2007],
["Flash Skeleton", "Flash [email protected]", 2008, 2006],
["The Admiral", "The [email protected]", 1998, 2005],
["The Tombstone", "The [email protected]", 2013, 2008],
["Golden Arrow", "Golden [email protected]", 2008, 2005],
["White Guardian", "White [email protected]", 2011, 2004],
["The Black Eastern Power", "The Black Eastern [email protected]", 2008, 2008],
["Ruthless Soldier", "Ruthless [email protected]", 2008, 2008],
["Dirty Clown", "Dirty [email protected]", 2008, 2008],
["Alpha Admiral", "Alpha [email protected]", 2008, 2008],
["Lightning Major", "Lightning [email protected]", 2008, 2008],
["The Rock Demon", "The Rock [email protected]", 2008, 2001],
["Wild Tiger", "Wild [email protected]", 2008, 2001],
["The Pointless Bandit", "The Pointless [email protected]", 2008, 2008],
["The Sergeant", "The [email protected]", 1998, 2002],
["Western Ogre", "Western [email protected]", 1998, 2004],
["Sergeant Strawberry", "Sergeant [email protected]", 2006, 2008],
]
)
def test_set_modifier_source(self):
def setValue(record, field, scope):
return 'Found same email address' \
if record.getField('email').getValue() in [ sr.getField('email').getValue() for sr in scope.getSourceRecords() ] \
else 'Not found same email address'
fn = setValue
source = StaticSource(
FieldSet([
Field('name', StringFieldType()),
Field('email', StringFieldType())
], FieldMap({
'name': 0,
'email': 1
})))
source.setResource(
[['El Agent', 'El [email protected]'],
['Ochala Wild', 'Ochala [email protected]'],
['Sina Venomous', 'Sina [email protected]'],
[
'Akassa Savage Phalloz',
'Akassa Savage [email protected]'
], ['Sermak Bad', 'Sermak [email protected]'],
['Olivia Deadly Dawod', 'Olivia Deadly [email protected]'],
['Pendus Inhuman', '*****@*****.**'],
[
'Naria Cold-blodded Greste',
'Naria Cold-blodded [email protected]'
], ['Shard Brutal', '*****@*****.**'],
['Sina Cruel', 'Sina [email protected]'],
['Deadly Ohmar', 'Deadly [email protected]'],
[
'Mylenedriz Cold-blodded',
'Mylenedriz [email protected]'
], ['Calden Frigid', 'Calden [email protected]'],
['Acid Reaper', '*****@*****.**'],
['Raven Seth', 'Raven [email protected]'],
['Random Leader', '*****@*****.**'],
['Pluto Brigadier', 'Pluto [email protected]'],
['Southern Kangaroo', 'Southern [email protected]'],
['Serious Flea', 'Serious [email protected]'],
['Nocturnal Raven', '*****@*****.**'],
['Risky Flea', 'Risky [email protected]'],
['Rivatha Todal', 'Rivatha [email protected]'],
['Panic Oliviaezit', 'Panic [email protected]'],
['Tomara Wild', 'Tomara [email protected]'],
['Venessa Metalhead', 'Venessa [email protected]'],
['Western Ogre', 'Western [email protected]'],
['Sergeant Strawberry', '*****@*****.**']])
modifier = SetModifier(self.reader,
fieldNames=['name'],
source=source,
fn=fn)
modifier.initialize()
records = [r for r in modifier.getRecords()]
modifier.finalize()
self.assertEqual(len(records), 85)
self.assertEqual(
len([
r for r in records
if r.getField('name').getValue() == 'Found same email address'
]), 6)
self.assertEqual(
len([
r for r in records if r.getField('name').getValue() ==
'Not found same email address'
]), 79)
class Test_Filter( unittest.TestCase ):
def setUp( self ):
self.reader = StaticSource(
FieldSet([
Field( 'name', StringFieldType() ),
Field( 'email', StringFieldType() ),
Field( 'year', IntegerFieldType() ),
Field( 'after_year', IntegerFieldType() )
],
FieldMap({
'name': 0,
'email': 1,
'year': 2,
'after_year': 3
}))
)
self.reader.setResource([
[ 'El Agent', 'El [email protected]', 2008, 2008 ],
[ 'Serious Electron', 'Serious [email protected]', 2008, 2013 ],
[ 'Brave Wizard', 'Brave [email protected]', 2008, 2008 ],
[ 'Forgotten Itchy Emperor', 'Forgotten Itchy [email protected]', 2008, 2013 ],
[ 'The Moving Monkey', 'The Moving [email protected]', 2008, 2008 ],
[ 'Evil Ghostly Brigadier', 'Evil Ghostly [email protected]', 2008, 2013 ],
[ 'Strangely Oyster', 'Strangely [email protected]', 2008, 2008 ],
[ 'Anaconda Silver', 'Anaconda [email protected]', 2006, 2008 ],
[ 'Hawk Tough', 'Hawk [email protected]', 2004, 2008 ],
[ 'The Disappointed Craw', 'The Disappointed [email protected]', 2008, 2013 ],
[ 'The Raven', 'The [email protected]', 1999, 2008 ],
[ 'Ruby Boomerang', 'Ruby [email protected]', 2008, 2008 ],
[ 'Skunk Tough', 'Skunk [email protected]', 2010, 2008 ],
[ 'The Nervous Forgotten Major', 'The Nervous Forgotten [email protected]', 2008, 2013 ],
[ 'Bursting Furious Puppet', 'Bursting Furious [email protected]', 2011, 2008 ],
[ 'Neptune Eagle', 'Neptune [email protected]', 2011, 2013 ],
[ 'The Skunk', 'The [email protected]', 2008, 2013 ],
[ 'Lone Demon', 'Lone [email protected]', 2008, 2008 ],
[ 'The Skunk', 'The [email protected]', 1999, 2008 ],
[ 'Gamma Serious Spear', 'Gamma Serious [email protected]', 2008, 2008 ],
[ 'Sleepy Dirty Sergeant', 'Sleepy Dirty [email protected]', 2008, 2008 ],
[ 'Red Monkey', 'Red [email protected]', 2008, 2008 ],
[ 'Striking Tiger', 'Striking [email protected]', 2005, 2008 ],
[ 'Sliding Demon', 'Sliding [email protected]', 2011, 2008 ],
[ 'Lone Commander', 'Lone [email protected]', 2008, 2013 ],
[ 'Dragon Insane', 'Dragon [email protected]', 2013, 2013 ],
[ 'Demon Skilled', 'Demon [email protected]', 2011, 2004 ],
[ 'Vulture Lucky', 'Vulture [email protected]', 2003, 2008 ],
[ 'The Ranger', 'The [email protected]', 2013, 2008 ],
[ 'Morbid Snake', 'Morbid [email protected]', 2011, 2008 ],
[ 'Dancing Skeleton', 'Dancing [email protected]', 2001, 2004 ],
[ 'The Psycho', 'The [email protected]', 2005, 2008 ],
[ 'Jupiter Rider', 'Jupiter [email protected]', 2011, 2008 ],
[ 'Green Dog', 'Green [email protected]', 2011, 2008 ],
[ 'Brutal Wild Colonel', 'Brutal Wild [email protected]', 2004, 2008 ],
[ 'Random Leader', 'Random [email protected]', 2008, 2008 ],
[ 'Pluto Brigadier', 'Pluto [email protected]', 2008, 2004 ],
[ 'Southern Kangaroo', 'Southern [email protected]', 2008, 2008 ],
[ 'Serious Flea', 'Serious [email protected]', 2001, 2005 ],
[ 'Nocturnal Raven', 'Nocturnal [email protected]', 2008, 2004 ],
[ 'Risky Flea', 'Risky [email protected]', 2005, 2005 ],
[ 'The Corporal', 'The [email protected]', 2013, 2008 ],
[ 'The Lucky Barbarian', 'The Lucky [email protected]', 2008, 2008 ],
[ 'Rocky Serious Dog', 'Rocky Serious [email protected]', 2008, 2008 ],
[ 'The Frozen Guardian', 'The Frozen [email protected]', 2008, 2008 ],
[ 'Freaky Frostbite', 'Freaky [email protected]', 2008, 2004 ],
[ 'The Tired Raven', 'The Tired [email protected]', 2008, 2008 ],
[ 'Disappointed Frostbite', 'Disappointed [email protected]', 2008, 2008 ],
[ 'The Craw', 'The [email protected]', 2003, 2008 ],
[ 'Gutsy Strangely Chief', 'Gutsy Strangely [email protected]', 2008, 2008 ],
[ 'Queen Angry', 'Queen [email protected]', 2008, 2008 ],
[ 'Pluto Albatross', 'Pluto [email protected]', 2003, 2008 ],
[ 'Endless Invader', 'Endless [email protected]', 2003, 2004 ],
[ 'Beta Young Sergeant', 'Beta Young [email protected]', 2008, 2011 ],
[ 'The Demon', 'The [email protected]', 2003, 2008 ],
[ 'Lone Monkey', 'Lone [email protected]', 2011, 2008 ],
[ 'Bursting Electron', 'Bursting [email protected]', 2003, 2010 ],
[ 'Gangster Solid', 'Gangster [email protected]', 2005, 2009 ],
[ 'The Gladiator', 'The [email protected]', 2001, 2002 ],
[ 'Flash Frostbite', 'Flash [email protected]', 2005, 2004 ],
[ 'The Rainbow Pluto Demon', 'The Rainbow Pluto [email protected]', 2011, 2013 ],
[ 'Poseidon Rider', 'Poseidon [email protected]', 2008, 2006 ],
[ 'The Old Alpha Brigadier', 'The Old Alpha [email protected]', 2008, 2008 ],
[ 'Rough Anaconda', 'Rough [email protected]', 2001, 2011 ],
[ 'Tough Dinosaur', 'Tough [email protected]', 2011, 2010 ],
[ 'The Lost Dinosaur', 'The Lost [email protected]', 2008, 2008 ],
[ 'The Raven', 'The [email protected]', 2005, 2009 ],
[ 'The Agent', 'The [email protected]', 2011, 2008 ],
[ 'Brave Scarecrow', 'Brave [email protected]', 2008, 2007 ],
[ 'Flash Skeleton', 'Flash [email protected]', 2008, 2006 ],
[ 'The Admiral', 'The [email protected]', 1998, 2005 ],
[ 'The Tombstone', 'The [email protected]', 2013, 2008 ],
[ 'Golden Arrow', 'Golden [email protected]', 2008, 2005 ],
[ 'White Guardian', 'White [email protected]', 2011, 2004 ],
[ 'The Black Eastern Power', 'The Black Eastern [email protected]', 2008, 2008 ],
[ 'Ruthless Soldier', 'Ruthless [email protected]', 2008, 2008 ],
[ 'Dirty Clown', 'Dirty [email protected]', 2008, 2008 ],
[ 'Alpha Admiral', 'Alpha [email protected]', 2008, 2008 ],
[ 'Lightning Major', 'Lightning [email protected]', 2008, 2008 ],
[ 'The Rock Demon', 'The Rock [email protected]', 2008, 2001 ],
[ 'Wild Tiger', 'Wild [email protected]', 2008, 2001 ],
[ 'The Pointless Bandit', 'The Pointless [email protected]', 2008, 2008 ],
[ 'The Sergeant', 'The [email protected]', 1998, 2002 ],
[ 'Western Ogre', 'Western [email protected]', 1998, 2004 ],
[ 'Sergeant Strawberry', 'Sergeant [email protected]', 2008, 2008 ]
])
self.other_source = StaticSource(
FieldSet([
Field( 'name', StringFieldType() ),
Field( 'email', StringFieldType() )
],
FieldMap({
'name': 0,
'email': 1
}))
)
self.other_source.setResource([
[ 'Ochala Wild', 'Ochala [email protected]' ],
[ 'Sina Venomous', 'Sina [email protected]' ],
[ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ],
[ 'Sermak Bad', 'Sermak [email protected]' ],
[ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ],
[ 'Pendus Inhuman', '*****@*****.**' ],
[ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ],
[ 'Shard Brutal', '*****@*****.**' ],
[ 'Sina Cruel', 'Sina [email protected]' ],
[ 'Deadly Ohmar', 'Deadly [email protected]' ],
[ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ],
[ 'Calden Frigid', 'Calden [email protected]' ],
[ 'Acid Reaper', '*****@*****.**' ],
[ 'Raven Seth', 'Raven [email protected]' ],
[ 'Random Leader', '*****@*****.**' ],
[ 'Pluto Brigadier', 'Pluto [email protected]' ],
[ 'Southern Kangaroo', 'Southern [email protected]' ],
[ 'Serious Flea', 'Serious [email protected]' ],
[ 'Nocturnal Raven', '*****@*****.**' ],
[ 'Risky Flea', 'Risky [email protected]' ],
[ 'Rivatha Todal', 'Rivatha [email protected]' ],
[ 'Panic Oliviaezit', 'Panic [email protected]' ],
[ 'Tomara Wild', 'Tomara [email protected]' ],
[ 'Venessa Metalhead', 'Venessa [email protected]' ],
[ 'Western Ogre', 'Western [email protected]' ],
[ 'Sergeant Strawberry', 'Sergeant [email protected]' ]
])
def test_drop_by_condition_filter( self ):
records = [ r for r in DropByConditionFilter(
self.reader,
condition = IsEqualCondition( 2008 ),
fieldNames = 'year'
).initialize().getRecords() ]
self.assertEqual( len( records ), 41 )
self.assertEqual( records[-1].getField('name').getValue(), 'Western Ogre' )
def test_drop_by_condition_filter_and( self ):
records = [ r for r in DropByConditionFilter(
self.reader,
condition = IsEqualCondition( 2008 ),
fieldNames = [ 'year', 'after_year' ]
).initialize().getRecords() ]
self.assertEqual( len( records ), 58 )
self.assertEqual( records[-1].getField('name').getValue(), 'Western Ogre' )
def test_drop_by_condition_filter_or( self ):
records = [ r for r in DropByConditionFilter(
self.reader,
condition = IsEqualCondition( 2008 ),
fieldNames = [ 'year', 'after_year' ],
operation = 'OR'
).initialize().getRecords() ]
self.assertEqual( len( records ), 19 )
self.assertEqual( records[-1].getField('name').getValue(), 'Western Ogre' )
def test_keep_by_condition_filter( self ):
records = [ r for r in KeepByConditionFilter(
self.reader,
condition = IsEqualCondition( 2008 ),
fieldNames = 'year'
).initialize().getRecords() ]
self.assertEqual( len( records ), 44 )
self.assertEqual( records[-1].getField('name').getValue(), 'Sergeant Strawberry' )
def test_keep_by_condition_filter_and( self ):
records = [ r for r in KeepByConditionFilter(
self.reader,
condition = IsEqualCondition( 2008 ),
fieldNames = [ 'year', 'after_year' ]
).initialize().getRecords() ]
self.assertEqual( len( records ), 27 )
self.assertEqual( records[-1].getField('name').getValue(), 'Sergeant Strawberry' )
def test_keep_by_condition_filter_or( self ):
records = [ r for r in KeepByConditionFilter(
self.reader,
condition = IsEqualCondition( 2008 ),
fieldNames = [ 'year', 'after_year' ],
operation = 'OR'
).initialize().getRecords() ]
self.assertEqual( len( records ), 66 )
self.assertEqual( records[-1].getField('name').getValue(), 'Sergeant Strawberry' )
def test_drop_field_filter( self ):
records = [ r for r in DropFieldFilter(
self.reader,
fieldNames = [ 'email', 'after_year' ]
).initialize().getRecords() ]
sourceRecords = [ r for r in self.reader.getRecords() ]
self.assertEqual( len( sourceRecords ), len( records ) )
self.assertEqual( len( records[0].getFieldNames() ), 2 )
self.assertEqual( records[0].getFieldNames()[-1], 'year' )
def test_drop_by_source_filter( self ):
f = DropBySourceFilter(
self.reader,
self.other_source,
condition = IsEqualCondition('email'),
join = 'name'
)
f.initialize()
records = [ r for r in f.getRecords() ]
f.finalize()
self.assertEqual( len( records ), 2 )
self.assertEqual( records[-1].getField('name').getValue(), 'Nocturnal Raven' )
class Test_Expand( unittest.TestCase ):
def setUp( self ):
self.reader = StaticSource(
FieldSet([
Field( 'name', StringFieldType() ),
Field( 'email', StringFieldType() )
],
FieldMap({
'name': 0,
'email': 1
}))
)
self.reader.setResource([
[ 'Ochala Wild', 'Ochala [email protected]' ],
[ 'Sina Venomous', 'Sina [email protected]' ],
[ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ],
[ 'Sermak Bad', 'Sermak [email protected]' ],
[ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ],
[ 'Pendus Inhuman', 'Pendus [email protected]' ],
[ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ],
[ 'Shard Brutal', 'Shard [email protected]' ],
[ 'Sina Cruel', 'Sina [email protected]' ],
[ 'Deadly Ohmar', 'Deadly [email protected]' ],
[ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ],
[ 'Calden Frigid', 'Calden [email protected]' ],
[ 'Acid Reaper', 'Acid [email protected]' ],
[ 'Raven Seth', 'Raven [email protected]' ],
[ 'Rivatha Todal', 'Rivatha [email protected]' ],
[ 'Panic Oliviaezit', 'Panic [email protected]' ],
[ 'Tomara Wild', 'Tomara [email protected]' ],
[ 'Venessa Metalhead', 'Venessa [email protected]' ]
])
def test_append_by_source( self ):
static_source = StaticSource(
FieldSet([
Field( 'name', StringFieldType() ),
Field( 'email', StringFieldType() ),
Field( 'year', IntegerFieldType() )
],
FieldMap({
'name': 0,
'email': 1,
'year': 2
}))
)
static_source.setResource([
[ 'El Agent', 'El [email protected]', 2008 ],
[ 'Serious Electron', 'Serious [email protected]', 2008 ],
[ 'Brave Wizard', 'Brave [email protected]', 2008 ],
[ 'Forgotten Itchy Emperor', 'Forgotten Itchy [email protected]', 2008 ],
[ 'The Moving Monkey', 'The Moving [email protected]', 2008 ],
[ 'Evil Ghostly Brigadier', 'Evil Ghostly [email protected]', 2008 ],
[ 'Strangely Oyster', 'Strangely [email protected]', 2008 ],
[ 'Anaconda Silver', 'Anaconda [email protected]', 2006 ],
[ 'Hawk Tough', 'Hawk [email protected]', 2004 ],
[ 'The Disappointed Craw', 'The Disappointed [email protected]', 2008 ],
[ 'The Raven', 'The [email protected]', 1999 ],
[ 'Ruby Boomerang', 'Ruby [email protected]', 2008 ],
[ 'Skunk Tough', 'Skunk [email protected]', 2010 ],
[ 'The Nervous Forgotten Major', 'The Nervous Forgotten [email protected]', 2008 ],
[ 'Bursting Furious Puppet', 'Bursting Furious [email protected]', 2011 ],
[ 'Neptune Eagle', 'Neptune [email protected]', 2011 ],
[ 'The Skunk', 'The [email protected]', 2008 ],
[ 'Lone Demon', 'Lone [email protected]', 2008 ],
[ 'The Skunk', 'The [email protected]', 1999 ],
[ 'Gamma Serious Spear', 'Gamma Serious [email protected]', 2008 ],
[ 'Sleepy Dirty Sergeant', 'Sleepy Dirty [email protected]', 2008 ],
[ 'Red Monkey', 'Red [email protected]', 2008 ],
[ 'Striking Tiger', 'Striking [email protected]', 2005 ],
[ 'Sliding Demon', 'Sliding [email protected]', 2011 ],
[ 'Lone Commander', 'Lone [email protected]', 2008 ],
[ 'Dragon Insane', 'Dragon [email protected]', 2013 ],
[ 'Demon Skilled', 'Demon [email protected]', 2011 ],
[ 'Vulture Lucky', 'Vulture [email protected]', 2003 ],
[ 'The Ranger', 'The [email protected]', 2013 ],
[ 'Morbid Snake', 'Morbid [email protected]', 2011 ],
[ 'Dancing Skeleton', 'Dancing [email protected]', 2001 ],
[ 'The Psycho', 'The [email protected]', 2005 ],
[ 'Jupiter Rider', 'Jupiter [email protected]', 2011 ],
[ 'Green Dog', 'Green [email protected]', 2011 ],
[ 'Brutal Wild Colonel', 'Brutal Wild [email protected]', 2008 ],
[ 'Random Leader', 'Random [email protected]', 2008 ],
[ 'Pluto Brigadier', 'Pluto [email protected]', 2008 ],
[ 'Southern Kangaroo', 'Southern [email protected]', 2008 ],
[ 'Serious Flea', 'Serious [email protected]', 2001 ],
[ 'Nocturnal Raven', 'Nocturnal [email protected]', 2008 ],
[ 'Risky Flea', 'Risky [email protected]', 2005 ],
[ 'The Corporal', 'The [email protected]', 2013 ],
[ 'The Lucky Barbarian', 'The Lucky [email protected]', 2008 ],
[ 'Rocky Serious Dog', 'Rocky Serious [email protected]', 2008 ],
[ 'The Frozen Guardian', 'The Frozen [email protected]', 2008 ],
[ 'Freaky Frostbite', 'Freaky [email protected]', 2008 ],
[ 'The Tired Raven', 'The Tired [email protected]', 2008 ],
[ 'Disappointed Frostbite', 'Disappointed [email protected]', 2008 ],
[ 'The Craw', 'The [email protected]', 2003 ],
[ 'Gutsy Strangely Chief', 'Gutsy Strangely [email protected]', 2008 ],
[ 'Queen Angry', 'Queen [email protected]', 2008 ],
[ 'Pluto Albatross', 'Pluto [email protected]', 2003 ],
[ 'Endless Invader', 'Endless [email protected]', 2003 ],
[ 'Beta Young Sergeant', 'Beta Young [email protected]', 2008 ],
[ 'The Demon', 'The [email protected]', 2003 ],
[ 'Lone Monkey', 'Lone [email protected]', 2011 ],
[ 'Bursting Electron', 'Bursting [email protected]', 2003 ],
[ 'Gangster Solid', 'Gangster [email protected]', 2005 ],
[ 'The Gladiator', 'The [email protected]', 2001 ],
[ 'Flash Frostbite', 'Flash [email protected]', 2005 ],
[ 'The Rainbow Pluto Demon', 'The Rainbow Pluto [email protected]', 2011 ],
[ 'Poseidon Rider', 'Poseidon [email protected]', 2008 ],
[ 'The Old Alpha Brigadier', 'The Old Alpha [email protected]', 2008 ],
[ 'Rough Anaconda', 'Rough [email protected]', 2001 ],
[ 'Tough Dinosaur', 'Tough [email protected]', 2011 ],
[ 'The Lost Dinosaur', 'The Lost [email protected]', 2008 ],
[ 'The Raven', 'The [email protected]', 2005 ],
[ 'The Agent', 'The [email protected]', 2011 ],
[ 'Brave Scarecrow', 'Brave [email protected]', 2008 ],
[ 'Flash Skeleton', 'Flash [email protected]', 2008 ],
[ 'The Admiral', 'The [email protected]', 1998 ],
[ 'The Tombstone', 'The [email protected]', 2013 ],
[ 'Golden Arrow', 'Golden [email protected]', 2008 ],
[ 'White Guardian', 'White [email protected]', 2011 ],
[ 'The Black Eastern Power', 'The Black Eastern [email protected]', 2008 ],
[ 'Ruthless Soldier', 'Ruthless [email protected]', 2008 ],
[ 'Dirty Clown', 'Dirty [email protected]', 2008 ],
[ 'Alpha Admiral', 'Alpha [email protected]', 2008 ],
[ 'Lightning Major', 'Lightning [email protected]', 2008 ],
[ 'The Rock Demon', 'The Rock [email protected]', 2008 ],
[ 'Wild Tiger', 'Wild [email protected]', 2008 ],
[ 'The Pointless Bandit', 'The Pointless [email protected]', 2008 ],
[ 'The Sergeant', 'The [email protected]', 1998 ],
[ 'Western Ogre', 'Western [email protected]', 1998 ],
[ 'Sergeant Strawberry', 'Sergeant [email protected]', 2008 ]
])
expand = AppendBySourceExpand( self.reader, static_source ).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 103 )
self.assertEqual( len( records[0].getFieldNames() ), 2 )
self.assertEqual( records[-1].getField('name').getValue(), 'Sergeant Strawberry' )
def test_append( self ):
source = CSVSource( FieldSet(
fields = [
Field( 'NUMBERS', StringFieldType() )
],
fieldmap = FieldMap({
'NUMBERS': 0
})
) )
source.setResource('tests/test_sources/test_csv_append.csv')
expand = AppendExpand( source, resource = 'tests/test_sources/test_csv_appended.csv' )
expand.initialize()
records = [ r for r in expand.getRecords() ]
expand.finalize()
self.assertEqual( len( records ), 6 )
self.assertEqual( records[0].getField('NUMBERS').getValue(), 'First' )
self.assertEqual( records[-1].getField('NUMBERS').getValue(), 'Sixth' )
def test_listexpander( self ):
static_source = StaticSource(
FieldSet([
Field( 'namelist', ListFieldType() ),
Field( 'email', StringFieldType() ),
Field( 'year', IntegerFieldType() ),
Field( 'name', StringFieldType() )
],
FieldMap({
'namelist': 0,
'email': 1,
'year': 2
}))
)
static_source.setResource([
[ list('El Agent'), 'El [email protected]', 2008 ],
[ list('Serious Electron'), 'Serious [email protected]', 2008 ],
[ list('Brave Wizard'), 'Brave [email protected]', 2008 ]
])
expand = ListExpanderExpand( static_source, 'namelist', 'name' ).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), len(list('El Agent')) + len(list('Serious Electron')) + len(list('Brave Wizard')) )
self.assertEqual( records[-1].getField('name').getValue(), 'd' )
self.assertEqual( records[-1].getField('email').getValue(), 'Brave [email protected]' )
def test_field( self ):
static_source = StaticSource(
FieldSet([
Field( 'year', IntegerFieldType() ),
Field( 'country', StringFieldType() ),
Field( 'count', IntegerFieldType() ),
Field( 'cz', IntegerFieldType() ),
Field( 'hu', IntegerFieldType() ),
Field( 'sk', IntegerFieldType() ),
Field( 'pl', IntegerFieldType() )
],
FieldMap({
'year': 0,
'cz': 1,
'hu': 2,
'sk': 3,
'pl': 4
}))
)
static_source.setResource([
[1999,32,694,129,230],
[1999,395,392,297,453],
[1999,635,812,115,97]
])
expand = FieldExpand(
static_source,
fieldNamesAndLabels = {
'cz': 'Czech',
'hu': 'Hungary',
'sk': 'Slovak',
'pl': 'Poland',
},
valueFieldName = 'count',
labelFieldName = 'country'
).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 12 )
self.assertEqual( records[-1].getField('year').getValue(), 1999 )
self.assertEqual( records[-1].getField('count').getValue(), 812 )
self.assertEqual( records[-1].getField('pl').getValue(), 97 )
self.assertEqual( records[-1].getField('country').getValue(), 'Hungary' )
def test_melt( self ):
static_source = StaticSource(
FieldSet([
Field( 'first', StringFieldType() ),
Field( 'height', FloatFieldType() ),
Field( 'last', StringFieldType() ),
Field( 'weight', IntegerFieldType() ),
Field( 'iq', IntegerFieldType() ),
Field( 'quantity', StringFieldType() ),
Field( 'value', FloatFieldType() )
],
FieldMap({
'first': 0,
'height': 1,
'last': 2,
'weight': 3,
'iq': 4
}))
)
static_source.setResource([
['John',5.5,'Doe',130,102],
['Mary',6.0,'Bo',150,98]
])
expand = MeltExpand(
static_source,
fieldNames = ['first','last'],
valueFieldName = 'value',
labelFieldName = 'quantity'
).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 6 )
self.assertEqual( len( records[0].getFieldNames() ), 4)
self.assertEqual( records[-1].getField('first').getValue(), 'Mary' )
self.assertEqual( records[-1].getField('last').getValue(), 'Bo' )
self.assertEqual( records[-1].getField('quantity').getValue(), 'height' )
self.assertEqual( records[-1].getField('value').getValue(), 6.0 )
def test_listexpander_map( self ):
static_source = StaticSource(
FieldSet([
Field( 'city', StringFieldType() ),
Field( 'geometry', ListFieldType() ),
Field( 'latitude', FloatFieldType() ),
Field( 'longitude', FloatFieldType() )
],
FieldMap({
'city': 0,
'geometry': 1
}))
)
static_source.setResource([
[ 'Balatonlelle', [[17.6874552,46.7871465],[17.6865955,46.7870049],[17.6846158,46.7866786],[17.6834977,46.7864944],[17.6822251,46.7862847],[17.6815319,46.7861705],[17.6811473,46.7861071],[17.6795989,46.785852],[17.6774482,46.7854976],[17.6739061,46.7849139],[17.6729351,46.7847539],[17.6720789,46.7846318]] ],
[ 'Balatonlelle', [[17.6871206,46.7880197],[17.6874552,46.7871465]] ],
[ 'Balatonboglar', "[[17.6709959,46.7843474],[17.6710995,46.7840514],[17.6711443,46.7838791],[17.6711725,46.7837746],[17.6713146,46.7831483]]" ]
])
expand = ListExpanderExpand( static_source, 'geometry', expanderMap = { 'latitude': 1, 'longitude': 0 } ).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 19 )
self.assertEqual( records[10].getField('city').getValue(), 'Balatonlelle' )
self.assertEqual( records[10].getField('latitude').getValue(), 46.7847539 )
self.assertEqual( records[-1].getField('city').getValue(), 'Balatonboglar' )
self.assertEqual( records[-1].getField('longitude').getValue(), 17.6713146 )
class Test_Filter(unittest.TestCase):
def setUp(self):
self.reader = StaticSource(
FieldSet([
Field('name', StringFieldType()),
Field('email', StringFieldType()),
Field('year', IntegerFieldType()),
Field('after_year', IntegerFieldType())
], FieldMap({
'name': 0,
'email': 1,
'year': 2,
'after_year': 3
})))
self.reader.setResource(
[['El Agent', 'El [email protected]', 2008, 2008],
[
'Serious Electron', 'Serious [email protected]',
2008, 2013
],
['Brave Wizard', 'Brave [email protected]', 2008, 2008],
[
'Forgotten Itchy Emperor',
'Forgotten Itchy [email protected]', 2008, 2013
],
[
'The Moving Monkey', 'The Moving [email protected]',
2008, 2008
],
[
'Evil Ghostly Brigadier',
'Evil Ghostly [email protected]', 2008, 2013
],
[
'Strangely Oyster', 'Strangely [email protected]',
2008, 2008
],
[
'Anaconda Silver', 'Anaconda [email protected]', 2006,
2008
], ['Hawk Tough', 'Hawk [email protected]', 2004, 2008],
[
'The Disappointed Craw',
'The Disappointed [email protected]', 2008, 2013
], ['The Raven', 'The [email protected]', 1999, 2008],
[
'Ruby Boomerang', 'Ruby [email protected]', 2008,
2008
], ['Skunk Tough', 'Skunk [email protected]', 2010, 2008],
[
'The Nervous Forgotten Major',
'The Nervous Forgotten [email protected]', 2008, 2013
],
[
'Bursting Furious Puppet',
'Bursting Furious [email protected]', 2011, 2008
],
['Neptune Eagle', 'Neptune [email protected]', 2011, 2013],
['The Skunk', 'The [email protected]', 2008, 2013],
['Lone Demon', 'Lone [email protected]', 2008, 2008],
['The Skunk', 'The [email protected]', 1999, 2008],
[
'Gamma Serious Spear',
'Gamma Serious [email protected]', 2008, 2008
],
[
'Sleepy Dirty Sergeant',
'Sleepy Dirty [email protected]', 2008, 2008
], ['Red Monkey', 'Red [email protected]', 2008, 2008],
[
'Striking Tiger', 'Striking [email protected]', 2005,
2008
],
['Sliding Demon', 'Sliding [email protected]', 2011, 2008],
[
'Lone Commander', 'Lone [email protected]', 2008,
2013
],
['Dragon Insane', 'Dragon [email protected]', 2013, 2013],
['Demon Skilled', 'Demon [email protected]', 2011, 2004],
['Vulture Lucky', 'Vulture [email protected]', 2003, 2008],
['The Ranger', 'The [email protected]', 2013, 2008],
['Morbid Snake', 'Morbid [email protected]', 2011, 2008],
[
'Dancing Skeleton', 'Dancing [email protected]',
2001, 2004
], ['The Psycho', 'The [email protected]', 2005, 2008],
['Jupiter Rider', 'Jupiter [email protected]', 2011, 2008],
['Green Dog', 'Green [email protected]', 2011, 2008],
[
'Brutal Wild Colonel',
'Brutal Wild [email protected]', 2004, 2008
],
['Random Leader', 'Random [email protected]', 2008, 2008],
[
'Pluto Brigadier', 'Pluto [email protected]', 2008,
2004
],
[
'Southern Kangaroo', 'Southern [email protected]',
2008, 2008
],
['Serious Flea', 'Serious [email protected]', 2001, 2005],
[
'Nocturnal Raven', 'Nocturnal [email protected]', 2008,
2004
], ['Risky Flea', 'Risky [email protected]', 2005, 2005],
['The Corporal', 'The [email protected]', 2013, 2008],
[
'The Lucky Barbarian',
'The Lucky [email protected]', 2008, 2008
],
[
'Rocky Serious Dog', 'Rocky Serious [email protected]',
2008, 2008
],
[
'The Frozen Guardian',
'The Frozen [email protected]', 2008, 2008
],
[
'Freaky Frostbite', 'Freaky [email protected]',
2008, 2004
],
[
'The Tired Raven', 'The Tired [email protected]', 2008,
2008
],
[
'Disappointed Frostbite',
'Disappointed [email protected]', 2008, 2008
], ['The Craw', 'The [email protected]', 2003, 2008],
[
'Gutsy Strangely Chief',
'Gutsy Strangely [email protected]', 2008, 2008
], ['Queen Angry', 'Queen [email protected]', 2008, 2008],
[
'Pluto Albatross', 'Pluto [email protected]', 2003,
2008
],
[
'Endless Invader', 'Endless [email protected]', 2003,
2004
],
[
'Beta Young Sergeant',
'Beta Young [email protected]', 2008, 2011
], ['The Demon', 'The [email protected]', 2003, 2008],
['Lone Monkey', 'Lone [email protected]', 2011, 2008],
[
'Bursting Electron', 'Bursting [email protected]',
2003, 2010
],
[
'Gangster Solid', 'Gangster [email protected]', 2005,
2009
],
['The Gladiator', 'The [email protected]', 2001, 2002],
[
'Flash Frostbite', 'Flash [email protected]', 2005,
2004
],
[
'The Rainbow Pluto Demon',
'The Rainbow Pluto [email protected]', 2011, 2013
],
[
'Poseidon Rider', 'Poseidon [email protected]', 2008,
2006
],
[
'The Old Alpha Brigadier',
'The Old Alpha [email protected]', 2008, 2008
],
[
'Rough Anaconda', 'Rough [email protected]', 2001,
2011
],
[
'Tough Dinosaur', 'Tough [email protected]', 2011,
2010
],
[
'The Lost Dinosaur', 'The Lost [email protected]',
2008, 2008
], ['The Raven', 'The [email protected]', 2005, 2009],
['The Agent', 'The [email protected]', 2011, 2008],
[
'Brave Scarecrow', 'Brave [email protected]', 2008,
2007
],
[
'Flash Skeleton', 'Flash [email protected]', 2008,
2006
], ['The Admiral', 'The [email protected]', 1998, 2005],
['The Tombstone', 'The [email protected]', 2013, 2008],
['Golden Arrow', 'Golden [email protected]', 2008, 2005],
[
'White Guardian', 'White [email protected]', 2011,
2004
],
[
'The Black Eastern Power',
'The Black Eastern [email protected]', 2008, 2008
],
[
'Ruthless Soldier', 'Ruthless [email protected]',
2008, 2008
], ['Dirty Clown', 'Dirty [email protected]', 2008, 2008],
['Alpha Admiral', 'Alpha [email protected]', 2008, 2008],
[
'Lightning Major', 'Lightning [email protected]', 2008,
2008
],
[
'The Rock Demon', 'The Rock [email protected]', 2008,
2001
], ['Wild Tiger', 'Wild [email protected]', 2008, 2001],
[
'The Pointless Bandit',
'The Pointless [email protected]', 2008, 2008
],
['The Sergeant', 'The [email protected]', 1998, 2002],
['Western Ogre', 'Western [email protected]', 1998, 2004],
[
'Sergeant Strawberry',
'Sergeant [email protected]', 2008, 2008
]])
self.other_source = StaticSource(
FieldSet([
Field('name', StringFieldType()),
Field('email', StringFieldType())
], FieldMap({
'name': 0,
'email': 1
})))
self.other_source.setResource(
[['Ochala Wild', 'Ochala [email protected]'],
['Sina Venomous', 'Sina [email protected]'],
[
'Akassa Savage Phalloz',
'Akassa Savage [email protected]'
], ['Sermak Bad', 'Sermak [email protected]'],
['Olivia Deadly Dawod', 'Olivia Deadly [email protected]'],
['Pendus Inhuman', '*****@*****.**'],
[
'Naria Cold-blodded Greste',
'Naria Cold-blodded [email protected]'
], ['Shard Brutal', '*****@*****.**'],
['Sina Cruel', 'Sina [email protected]'],
['Deadly Ohmar', 'Deadly [email protected]'],
[
'Mylenedriz Cold-blodded',
'Mylenedriz [email protected]'
], ['Calden Frigid', 'Calden [email protected]'],
['Acid Reaper', '*****@*****.**'],
['Raven Seth', 'Raven [email protected]'],
['Random Leader', '*****@*****.**'],
['Pluto Brigadier', 'Pluto [email protected]'],
['Southern Kangaroo', 'Southern [email protected]'],
['Serious Flea', 'Serious [email protected]'],
['Nocturnal Raven', '*****@*****.**'],
['Risky Flea', 'Risky [email protected]'],
['Rivatha Todal', 'Rivatha [email protected]'],
['Panic Oliviaezit', 'Panic [email protected]'],
['Tomara Wild', 'Tomara [email protected]'],
['Venessa Metalhead', 'Venessa [email protected]'],
['Western Ogre', 'Western [email protected]'],
['Sergeant Strawberry',
'Sergeant [email protected]']])
def test_logger_filter_default(self):
filtr = DropByConditionFilter(self.reader,
condition=IsEqualCondition(2008),
fieldNames='year')
filtr.setLogFile(logFile='tests/logger', appendLog=False, logger=None)
filtr.initialize()
records = [r for r in filtr.getRecords()]
filtr.finalize()
hashcode = hashlib.md5(open('tests/logger', 'rb').read()).hexdigest()
os.unlink('tests/logger')
self.assertEqual(hashcode, '95eff99740e2d8d2b50808357e006293')
def test_logger_filter_fn(self):
filtr = DropByConditionFilter(self.reader,
condition=IsEqualCondition(2008),
fieldNames='year')
filtr.setLogFile(logFile='tests/logger',
appendLog=False,
logger='tests.test_filter.notDroppedLogger')
filtr.initialize()
records = [r for r in filtr.getRecords()]
filtr.finalize()
hashcode = hashlib.md5(open('tests/logger', 'rb').read()).hexdigest()
os.unlink('tests/logger')
self.assertEqual(hashcode, '3c30fe720f1b410fce6cf17651978b13')
def test_drop_by_condition_filter(self):
records = [
r for r in DropByConditionFilter(
self.reader,
condition=IsEqualCondition(2008),
fieldNames='year').initialize().getRecords()
]
self.assertEqual(len(records), 41)
self.assertEqual(records[-1].getField('name').getValue(),
'Western Ogre')
def test_drop_by_condition_filter_and(self):
records = [
r for r in DropByConditionFilter(
self.reader,
condition=IsEqualCondition(2008),
fieldNames=['year', 'after_year']).initialize().getRecords()
]
self.assertEqual(len(records), 58)
self.assertEqual(records[-1].getField('name').getValue(),
'Western Ogre')
def test_drop_by_condition_filter_or(self):
records = [
r for r in DropByConditionFilter(
self.reader,
condition=IsEqualCondition(2008),
fieldNames=['year', 'after_year'],
operation='OR').initialize().getRecords()
]
self.assertEqual(len(records), 19)
self.assertEqual(records[-1].getField('name').getValue(),
'Western Ogre')
def test_keep_by_condition_filter(self):
records = [
r for r in KeepByConditionFilter(
self.reader,
condition=IsEqualCondition(2008),
fieldNames='year').initialize().getRecords()
]
self.assertEqual(len(records), 44)
self.assertEqual(records[-1].getField('name').getValue(),
'Sergeant Strawberry')
def test_keep_by_condition_filter_and(self):
records = [
r for r in KeepByConditionFilter(
self.reader,
condition=IsEqualCondition(2008),
fieldNames=['year', 'after_year']).initialize().getRecords()
]
self.assertEqual(len(records), 27)
self.assertEqual(records[-1].getField('name').getValue(),
'Sergeant Strawberry')
def test_keep_by_condition_filter_or(self):
records = [
r for r in KeepByConditionFilter(
self.reader,
condition=IsEqualCondition(2008),
fieldNames=['year', 'after_year'],
operation='OR').initialize().getRecords()
]
self.assertEqual(len(records), 66)
self.assertEqual(records[-1].getField('name').getValue(),
'Sergeant Strawberry')
def test_drop_field_filter(self):
records = [
r
for r in DropFieldFilter(self.reader,
fieldNames=['email', 'after_year'
]).initialize().getRecords()
]
sourceRecords = [r for r in self.reader.getRecords()]
self.assertEqual(len(sourceRecords), len(records))
self.assertEqual(len(records[0].getFieldNames()), 2)
self.assertEqual(records[0].getFieldNames()[-1], 'year')
def test_drop_by_source_filter(self):
f = DropBySourceFilter(self.reader,
self.other_source,
condition=IsEqualCondition('email'),
join='name')
f.initialize()
records = [r for r in f.getRecords()]
f.finalize()
self.assertEqual(len(records), 2)
self.assertEqual(records[-1].getField('name').getValue(),
'Nocturnal Raven')
def setUp(self):
self.reader = StaticSource(
FieldSet([
Field('name', StringFieldType()),
Field('email', StringFieldType()),
Field('year', IntegerFieldType()),
Field('after_year', IntegerFieldType()),
Field('number', IntegerFieldType())
], FieldMap({
'name': 0,
'email': 1,
'year': 2,
'after_year': 3
})))
self.reader.setResource(
[['El Agent', 'El [email protected]', 2008, 2008],
[
'Serious Electron', 'Serious [email protected]',
2008, 2013
],
['Brave Wizard', 'Brave [email protected]', 2008, 2008],
[
'Forgotten Itchy Emperor',
'Forgotten Itchy [email protected]', 2008, 2013
],
[
'The Moving Monkey', 'The Moving [email protected]',
2008, 2008
],
[
'Evil Ghostly Brigadier',
'Evil Ghostly [email protected]', 2008, 2013
],
[
'Strangely Oyster', 'Strangely [email protected]',
2008, 2008
],
[
'Anaconda Silver', 'Anaconda [email protected]', 2006,
2008
], ['Hawk Tough', 'Hawk [email protected]', 2004, 2008],
[
'The Disappointed Craw',
'The Disappointed [email protected]', 2008, 2013
], ['The Raven', 'The [email protected]', 1999, 2008],
[
'Ruby Boomerang', 'Ruby [email protected]', 2008,
2008
], ['Skunk Tough', 'Skunk [email protected]', 2010, 2008],
[
'The Nervous Forgotten Major',
'The Nervous Forgotten [email protected]', 2008, 2013
],
[
'Bursting Furious Puppet',
'Bursting Furious [email protected]', 2011, 2008
],
['Neptune Eagle', 'Neptune [email protected]', 2011, 2013],
['The Skunk', 'The [email protected]', 2008, 2013],
['Lone Demon', 'Lone [email protected]', 2008, 2008],
['The Skunk', 'The [email protected]', 1999, 2008],
[
'Gamma Serious Spear',
'Gamma Serious [email protected]', 2008, 2008
],
[
'Sleepy Dirty Sergeant',
'Sleepy Dirty [email protected]', 2008, 2008
], ['Red Monkey', 'Red [email protected]', 2008, 2008],
[
'Striking Tiger', 'Striking [email protected]', 2005,
2008
],
['Sliding Demon', 'Sliding [email protected]', 2011, 2008],
[
'Lone Commander', 'Lone [email protected]', 2008,
2013
],
['Dragon Insane', 'Dragon [email protected]', 2013, 2013],
['Demon Skilled', 'Demon [email protected]', 2011, 2004],
['Vulture Lucky', 'Vulture [email protected]', 2003, 2008],
['The Ranger', 'The [email protected]', 2013, 2008],
['Morbid Snake', 'Morbid [email protected]', 2011, 2008],
[
'Dancing Skeleton', 'Dancing [email protected]',
2001, 2004
], ['The Psycho', 'The [email protected]', 2005, 2008],
['Jupiter Rider', 'Jupiter [email protected]', 2011, 2008],
['Green Dog', 'Green [email protected]', 2011, 2008],
[
'Brutal Wild Colonel',
'Brutal Wild [email protected]', 2004, 2008
],
['Random Leader', 'Random [email protected]', 2008, 2008],
[
'Pluto Brigadier', 'Pluto [email protected]', 2008,
2004
],
[
'Southern Kangaroo', 'Southern [email protected]',
2008, 2008
],
['Serious Flea', 'Serious [email protected]', 2001, 2005],
[
'Nocturnal Raven', 'Nocturnal [email protected]', 2008,
2004
], ['Risky Flea', 'Risky [email protected]', 2005, 2005],
['The Corporal', 'The [email protected]', 2013, 2008],
[
'The Lucky Barbarian',
'The Lucky [email protected]', 2008, 2008
],
[
'Rocky Serious Dog', 'Rocky Serious [email protected]',
2008, 2008
],
[
'The Frozen Guardian',
'The Frozen [email protected]', 2008, 2008
],
[
'Freaky Frostbite', 'Freaky [email protected]',
2008, 2004
],
[
'The Tired Raven', 'The Tired [email protected]', 2008,
2008
],
[
'Disappointed Frostbite',
'Disappointed [email protected]', 2008, 2008
], ['The Craw', 'The [email protected]', 2003, 2008],
[
'Gutsy Strangely Chief',
'Gutsy Strangely [email protected]', 2008, 2008
], ['Queen Angry', 'Queen [email protected]', 2008, 2008],
[
'Pluto Albatross', 'Pluto [email protected]', 2003,
2008
],
[
'Endless Invader', 'Endless [email protected]', 2003,
2004
],
[
'Beta Young Sergeant',
'Beta Young [email protected]', 2008, 2011
], ['The Demon', 'The [email protected]', 2003, 2008],
['Lone Monkey', 'Lone [email protected]', 2011, 2008],
[
'Bursting Electron', 'Bursting [email protected]',
2003, 2010
],
[
'Gangster Solid', 'Gangster [email protected]', 2005,
2009
],
['The Gladiator', 'The [email protected]', 2001, 2002],
[
'Flash Frostbite', 'Flash [email protected]', 2005,
2004
],
[
'The Rainbow Pluto Demon',
'The Rainbow Pluto [email protected]', 2011, 2013
],
[
'Poseidon Rider', 'Poseidon [email protected]', 2008,
2006
],
[
'The Old Alpha Brigadier',
'The Old Alpha [email protected]', 2008, 2008
],
[
'Rough Anaconda', 'Rough [email protected]', 2001,
2011
],
[
'Tough Dinosaur', 'Tough [email protected]', 2011,
2010
],
[
'The Lost Dinosaur', 'The Lost [email protected]',
2008, 2008
], ['The Raven', 'The [email protected]', 2005, 2009],
['The Agent', 'The [email protected]', 2011, 2008],
[
'Brave Scarecrow', 'Brave [email protected]', 2008,
2007
],
[
'Flash Skeleton', 'Flash [email protected]', 2008,
2006
], ['The Admiral', 'The [email protected]', 1998, 2005],
['The Tombstone', 'The [email protected]', 2013, 2008],
['Golden Arrow', 'Golden [email protected]', 2008, 2005],
[
'White Guardian', 'White [email protected]', 2011,
2004
],
[
'The Black Eastern Power',
'The Black Eastern [email protected]', 2008, 2008
],
[
'Ruthless Soldier', 'Ruthless [email protected]',
2008, 2008
], ['Dirty Clown', 'Dirty [email protected]', 2008, 2008],
['Alpha Admiral', 'Alpha [email protected]', 2008, 2008],
[
'Lightning Major', 'Lightning [email protected]', 2008,
2008
],
[
'The Rock Demon', 'The Rock [email protected]', 2008,
2001
], ['Wild Tiger', 'Wild [email protected]', 2008, 2001],
[
'The Pointless Bandit',
'The Pointless [email protected]', 2008, 2008
],
['The Sergeant', 'The [email protected]', 1998, 2002],
['Western Ogre', 'Western [email protected]', 1998, 2004],
[
'Sergeant Strawberry',
'Sergeant [email protected]', 2006, 2008
]])
class Test_Modifier(unittest.TestCase):
def setUp(self):
self.reader = StaticSource(
FieldSet([
Field('name', StringFieldType()),
Field('email', StringFieldType()),
Field('year', IntegerFieldType()),
Field('after_year', IntegerFieldType()),
Field('age', IntegerFieldType())
], FieldMap({
'name': 0,
'email': 1,
'year': 2,
'after_year': 3
})))
self.reader.setResource(
[['El Agent', 'El [email protected]', 2008, 2008],
[
'Serious Electron', 'Serious [email protected]',
2008, 2013
],
['Brave Wizard', 'Brave [email protected]', 2008, 2008],
[
'Forgotten Itchy Emperor',
'Forgotten Itchy [email protected]', 2008, 2013
],
[
'The Moving Monkey', 'The Moving [email protected]',
2008, 2008
],
[
'Evil Ghostly Brigadier',
'Evil Ghostly [email protected]', 2008, 2013
],
[
'Strangely Oyster', 'Strangely [email protected]',
2008, 2008
],
[
'Anaconda Silver', 'Anaconda [email protected]', 2006,
2008
], ['Hawk Tough', 'Hawk [email protected]', 2004, 2008],
[
'The Disappointed Craw',
'The Disappointed [email protected]', 2008, 2013
], ['The Raven', 'The [email protected]', 1999, 2008],
[
'Ruby Boomerang', 'Ruby [email protected]', 2008,
2008
], ['Skunk Tough', 'Skunk [email protected]', 2010, 2008],
[
'The Nervous Forgotten Major',
'The Nervous Forgotten [email protected]', 2008, 2013
],
[
'Bursting Furious Puppet',
'Bursting Furious [email protected]', 2011, 2008
],
['Neptune Eagle', 'Neptune [email protected]', 2011, 2013],
['The Skunk', 'The [email protected]', 2008, 2013],
['Lone Demon', 'Lone [email protected]', 2008, 2008],
['The Skunk', 'The [email protected]', 1999, 2008],
[
'Gamma Serious Spear',
'Gamma Serious [email protected]', 2008, 2008
],
[
'Sleepy Dirty Sergeant',
'Sleepy Dirty [email protected]', 2008, 2008
], ['Red Monkey', 'Red [email protected]', 2008, 2008],
[
'Striking Tiger', 'Striking [email protected]', 2005,
2008
],
['Sliding Demon', 'Sliding [email protected]', 2011, 2008],
[
'Lone Commander', 'Lone [email protected]', 2008,
2013
],
['Dragon Insane', 'Dragon [email protected]', 2013, 2013],
['Demon Skilled', 'Demon [email protected]', 2011, 2004],
['Vulture Lucky', 'Vulture [email protected]', 2003, 2008],
['The Ranger', 'The [email protected]', 2013, 2008],
['Morbid Snake', 'Morbid [email protected]', 2011, 2008],
[
'Dancing Skeleton', 'Dancing [email protected]',
2001, 2004
], ['The Psycho', 'The [email protected]', 2005, 2008],
['Jupiter Rider', 'Jupiter [email protected]', 2011, 2008],
['Green Dog', 'Green [email protected]', 2011, 2008],
[
'Brutal Wild Colonel',
'Brutal Wild [email protected]', 2004, 2008
],
['Random Leader', 'Random [email protected]', 2008, 2008],
[
'Pluto Brigadier', 'Pluto [email protected]', 2008,
2004
],
[
'Southern Kangaroo', 'Southern [email protected]',
2008, 2008
],
['Serious Flea', 'Serious [email protected]', 2001, 2005],
[
'Nocturnal Raven', 'Nocturnal [email protected]', 2008,
2004
], ['Risky Flea', 'Risky [email protected]', 2005, 2005],
['The Corporal', 'The [email protected]', 2013, 2008],
[
'The Lucky Barbarian',
'The Lucky [email protected]', 2008, 2008
],
[
'Rocky Serious Dog', 'Rocky Serious [email protected]',
2008, 2008
],
[
'The Frozen Guardian',
'The Frozen [email protected]', 2008, 2008
],
[
'Freaky Frostbite', 'Freaky [email protected]',
2008, 2004
],
[
'The Tired Raven', 'The Tired [email protected]', 2008,
2008
],
[
'Disappointed Frostbite',
'Disappointed [email protected]', 2008, 2008
], ['The Craw', 'The [email protected]', 2003, 2008],
[
'Gutsy Strangely Chief',
'Gutsy Strangely [email protected]', 2008, 2008
], ['Queen Angry', 'Queen [email protected]', 2008, 2008],
[
'Pluto Albatross', 'Pluto [email protected]', 2003,
2008
],
[
'Endless Invader', 'Endless [email protected]', 2003,
2004
],
[
'Beta Young Sergeant',
'Beta Young [email protected]', 2008, 2011
], ['The Demon', 'The [email protected]', 2003, 2008],
['Lone Monkey', 'Lone [email protected]', 2011, 2008],
[
'Bursting Electron', 'Bursting [email protected]',
2003, 2010
],
[
'Gangster Solid', 'Gangster [email protected]', 2005,
2009
],
['The Gladiator', 'The [email protected]', 2001, 2002],
[
'Flash Frostbite', 'Flash [email protected]', 2005,
2004
],
[
'The Rainbow Pluto Demon',
'The Rainbow Pluto [email protected]', 2011, 2013
],
[
'Poseidon Rider', 'Poseidon [email protected]', 2008,
2006
],
[
'The Old Alpha Brigadier',
'The Old Alpha [email protected]', 2008, 2008
],
[
'Rough Anaconda', 'Rough [email protected]', 2001,
2011
],
[
'Tough Dinosaur', 'Tough [email protected]', 2011,
2010
],
[
'The Lost Dinosaur', 'The Lost [email protected]',
2008, 2008
], ['The Raven', 'The [email protected]', 2005, 2009],
['The Agent', 'The [email protected]', 2011, 2008],
[
'Brave Scarecrow', 'Brave [email protected]', 2008,
2007
],
[
'Flash Skeleton', 'Flash [email protected]', 2008,
2006
], ['The Admiral', 'The [email protected]', 1998, 2005],
['The Tombstone', 'The [email protected]', 2013, 2008],
['Golden Arrow', 'Golden [email protected]', 2008, 2005],
[
'White Guardian', 'White [email protected]', 2011,
2004
],
[
'The Black Eastern Power',
'The Black Eastern [email protected]', 2008, 2008
],
[
'Ruthless Soldier', 'Ruthless [email protected]',
2008, 2008
], ['Dirty Clown', 'Dirty [email protected]', 2008, 2008],
['Alpha Admiral', 'Alpha [email protected]', 2008, 2008],
[
'Lightning Major', 'Lightning [email protected]', 2008,
2008
],
[
'The Rock Demon', 'The Rock [email protected]', 2008,
2001
], ['Wild Tiger', 'Wild [email protected]', 2008, 2001],
[
'The Pointless Bandit',
'The Pointless [email protected]', 2008, 2008
],
['The Sergeant', 'The [email protected]', 1998, 2002],
['Western Ogre', 'Western [email protected]', 1998, 2004],
[
'Sergeant Strawberry',
'Sergeant [email protected]', 2006, 2008
]])
def test_joinbykey_modifier(self):
source = StaticSource(
FieldSet([
Field('email', StringFieldType(), key=True),
Field('age', IntegerFieldType())
], FieldMap({
'email': 0,
'age': 1
})))
source.setResource(
[['El [email protected]', 12],
['Ochala [email protected]', 14],
['Sina [email protected]', 17],
['Akassa Savage [email protected]', 16],
['Sermak [email protected]', 22],
['Olivia Deadly [email protected]', 32],
['*****@*****.**', 42],
['Naria Cold-blodded [email protected]', 22],
['*****@*****.**', 54],
['Sina [email protected]', 56],
['Deadly [email protected]', 43],
['Mylenedriz [email protected]', 23],
['Calden [email protected]', 35],
['*****@*****.**', 56],
['Raven [email protected]', 23],
['*****@*****.**', 45],
['Pluto [email protected]', 64],
['Southern [email protected]', 53],
['Serious [email protected]', 62],
['*****@*****.**', 63],
['Risky [email protected]', 21],
['Rivatha [email protected]', 56],
['Panic [email protected]', 25],
['Tomara [email protected]', 46],
['Venessa [email protected]', 53],
['Western [email protected]', 71],
['*****@*****.**', 76]])
records = [
r for r in
JoinByKeyModifier(self.reader, fieldNames=['age'],
source=source).initialize().getRecords()
]
self.assertEqual(len(records), 85)
self.assertIsNone(records[-1].getField('age').getValue())
self.assertEqual(records[-2].getField('age').getValue(), 71)
self.assertEqual(records[0].getField('age').getValue(), 12)
def test_order_modifier(self):
records = [
r for r in OrderModifier(self.reader,
fieldNamesAndOrder=[{
'year': 'DESC'
}, {
'name': 'ASC'
}]).initialize().getRecords()
]
self.assertEqual(len(records), 85)
self.assertEqual(records[-1].getField('year').getValue(), 1998)
self.assertEqual(records[-4].getField('year').getValue(), 1999)
self.assertEqual(records[0].getField('year').getValue(), 2013)
self.assertEqual(records[-1].getField('name').getValue(),
'Western Ogre')
self.assertEqual(records[-2].getField('name').getValue(),
'The Sergeant')
self.assertEqual(records[-3].getField('name').getValue(),
'The Admiral')
self.assertEqual(records[-4].getField('name').getValue(), 'The Skunk')
self.assertEqual(records[0].getField('name').getValue(),
'Dragon Insane')
def test_transform_field_modifier(self):
records = [
r for r in TransformFieldModifier(
self.reader,
fieldNames=['email'],
transforms=[ReplaceByRegexpTransform(regexp=u'[ ]+', to=u'')
]).initialize().getRecords()
]
self.assertEqual(len(records), 85)
self.assertEqual(records[-1].getField('email').getValue(),
'*****@*****.**')
def test_set_modifier_without_param(self):
records = [
r for r in SetModifier(
self.reader, fieldNames=['name']).initialize().getRecords()
]
self.assertEqual(len(records), 85)
self.assertEqual(
len([
r for r in records if r.getField('name').getValue() is not None
]), 0)
def test_set_modifier_with_sprintf_value(self):
records = [
r for r in SetModifier(
self.reader,
fieldNames=['email'],
value='%(name)s <%(email)s>').initialize().getRecords()
]
self.assertEqual(len(records), 85)
self.assertEqual(
records[-1].getField('email').getValue(),
'Sergeant Strawberry <Sergeant [email protected]>')
def test_set_modifier_value(self):
records = [
r for r in SetModifier(
self.reader, fieldNames=['name'],
value='Same name for all').initialize().getRecords()
]
self.assertEqual(len(records), 85)
self.assertEqual(records[-1].getField('name').getValue(),
'Same name for all')
def test_set_modifier_fn(self):
def setValue(record, field, scope):
if record.getField('year').getValue() == 2008:
return 'Same name for 2008 years'
return 'Other name for anything else'
fn = setValue
records = [
r for r in SetModifier(
self.reader,
fieldNames=['name'],
fn=fn,
).initialize().getRecords()
]
self.assertEqual(len(records), 85)
self.assertEqual(records[-1].getField('name').getValue(),
'Other name for anything else')
self.assertEqual(records[0].getField('name').getValue(),
'Same name for 2008 years')
self.assertEqual(
len([
r for r in records
if r.getField('name').getValue() == 'Same name for 2008 years'
]), 43)
self.assertEqual(
len([
r for r in records if r.getField('name').getValue() ==
'Other name for anything else'
]), 42)
def test_set_modifier_source(self):
def setValue(record, field, scope):
return 'Found same email address' \
if record.getField('email').getValue() in [ sr.getField('email').getValue() for sr in scope.getSourceRecords() ] \
else 'Not found same email address'
fn = setValue
source = StaticSource(
FieldSet([
Field('name', StringFieldType()),
Field('email', StringFieldType())
], FieldMap({
'name': 0,
'email': 1
})))
source.setResource(
[['El Agent', 'El [email protected]'],
['Ochala Wild', 'Ochala [email protected]'],
['Sina Venomous', 'Sina [email protected]'],
[
'Akassa Savage Phalloz',
'Akassa Savage [email protected]'
], ['Sermak Bad', 'Sermak [email protected]'],
['Olivia Deadly Dawod', 'Olivia Deadly [email protected]'],
['Pendus Inhuman', '*****@*****.**'],
[
'Naria Cold-blodded Greste',
'Naria Cold-blodded [email protected]'
], ['Shard Brutal', '*****@*****.**'],
['Sina Cruel', 'Sina [email protected]'],
['Deadly Ohmar', 'Deadly [email protected]'],
[
'Mylenedriz Cold-blodded',
'Mylenedriz [email protected]'
], ['Calden Frigid', 'Calden [email protected]'],
['Acid Reaper', '*****@*****.**'],
['Raven Seth', 'Raven [email protected]'],
['Random Leader', '*****@*****.**'],
['Pluto Brigadier', 'Pluto [email protected]'],
['Southern Kangaroo', 'Southern [email protected]'],
['Serious Flea', 'Serious [email protected]'],
['Nocturnal Raven', '*****@*****.**'],
['Risky Flea', 'Risky [email protected]'],
['Rivatha Todal', 'Rivatha [email protected]'],
['Panic Oliviaezit', 'Panic [email protected]'],
['Tomara Wild', 'Tomara [email protected]'],
['Venessa Metalhead', 'Venessa [email protected]'],
['Western Ogre', 'Western [email protected]'],
['Sergeant Strawberry', '*****@*****.**']])
modifier = SetModifier(self.reader,
fieldNames=['name'],
source=source,
fn=fn)
modifier.initialize()
records = [r for r in modifier.getRecords()]
modifier.finalize()
self.assertEqual(len(records), 85)
self.assertEqual(
len([
r for r in records
if r.getField('name').getValue() == 'Found same email address'
]), 6)
self.assertEqual(
len([
r for r in records if r.getField('name').getValue() ==
'Not found same email address'
]), 79)
