def setUp(self):
self.fs = FieldSet([
Field('id', IntegerFieldType(), defaultValue=1, key=True),
Field('name', StringFieldType(), defaultValue='John doe'),
Field('email', StringFieldType(), key=True),
Field('birth_date',
DateFieldType(),
field_final_type=StringFieldType()),
Field('favourite_number', FloatFieldType()),
Field('created',
DateTimeFieldType(),
defaultValue='2013-04-04 18:44:32'),
Field('updated',
DateTimeFieldType(),
defaultValue='2013-04-04 18:44:32')
])
self.fm = FieldMap({
'id': 0,
'name': 1,
'email': 2,
'birth_date': 3,
'favourite_number': 4,
'created': 5,
'updated': 6,
'not_existing_field_name': 6
})
def test_remove_words_by_source(self):
sourceRecords = [
FieldSet(
[Field('name', StringFieldType(), defaultValue='killing')]),
FieldSet(
[Field('name', StringFieldType(), defaultValue='sadness')]),
FieldSet(
[Field('name', StringFieldType(), defaultValue='hungarian')])
]
self.field.setValue(
u'contains hungarian members attractive sadness killing')
self.field.setTransforms([RemoveWordsBySourceTransform(sourceRecords)
]).run()
self.assertEqual(self.field.getValue(), 'contains members attractive')
def setUp( self ):
self.reader = StaticSource(
FieldSet([
Field( 'name', StringFieldType() ),
Field( 'email', StringFieldType() )
],
FieldMap({
'name': 0,
'email': 1
}))
)
self.reader.setResource([
[ 'Ochala Wild', 'Ochala [email protected]' ],
[ 'Sina Venomous', 'Sina [email protected]' ],
[ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ],
[ 'Sermak Bad', 'Sermak [email protected]' ],
[ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ],
[ 'Pendus Inhuman', 'Pendus [email protected]' ],
[ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ],
[ 'Shard Brutal', 'Shard [email protected]' ],
[ 'Sina Cruel', 'Sina [email protected]' ],
[ 'Deadly Ohmar', 'Deadly [email protected]' ],
[ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ],
[ 'Calden Frigid', 'Calden [email protected]' ],
[ 'Acid Reaper', 'Acid [email protected]' ],
[ 'Raven Seth', 'Raven [email protected]' ],
[ 'Rivatha Todal', 'Rivatha [email protected]' ],
[ 'Panic Oliviaezit', 'Panic [email protected]' ],
[ 'Tomara Wild', 'Tomara [email protected]' ],
[ 'Venessa Metalhead', 'Venessa [email protected]' ]
])
def test_listexpander_map( self ):
static_source = StaticSource(
FieldSet([
Field( 'city', StringFieldType() ),
Field( 'geometry', ListFieldType() ),
Field( 'latitude', FloatFieldType() ),
Field( 'longitude', FloatFieldType() )
],
FieldMap({
'city': 0,
'geometry': 1
}))
)
static_source.setResource([
[ 'Balatonlelle', [[17.6874552,46.7871465],[17.6865955,46.7870049],[17.6846158,46.7866786],[17.6834977,46.7864944],[17.6822251,46.7862847],[17.6815319,46.7861705],[17.6811473,46.7861071],[17.6795989,46.785852],[17.6774482,46.7854976],[17.6739061,46.7849139],[17.6729351,46.7847539],[17.6720789,46.7846318]] ],
[ 'Balatonlelle', [[17.6871206,46.7880197],[17.6874552,46.7871465]] ],
[ 'Balatonboglar', "[[17.6709959,46.7843474],[17.6710995,46.7840514],[17.6711443,46.7838791],[17.6711725,46.7837746],[17.6713146,46.7831483]]" ]
])
expand = ListExpanderExpand( static_source, 'geometry', expanderMap = { 'latitude': 1, 'longitude': 0 } ).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 19 )
self.assertEqual( records[10].getField('city').getValue(), 'Balatonlelle' )
self.assertEqual( records[10].getField('latitude').getValue(), 46.7847539 )
self.assertEqual( records[-1].getField('city').getValue(), 'Balatonboglar' )
self.assertEqual( records[-1].getField('longitude').getValue(), 17.6713146 )
def test_listexpander( self ):
static_source = StaticSource(
FieldSet([
Field( 'namelist', ListFieldType() ),
Field( 'email', StringFieldType() ),
Field( 'year', IntegerFieldType() ),
Field( 'name', StringFieldType() )
],
FieldMap({
'namelist': 0,
'email': 1,
'year': 2
}))
)
static_source.setResource([
[ list('El Agent'), 'El [email protected]', 2008 ],
[ list('Serious Electron'), 'Serious [email protected]', 2008 ],
[ list('Brave Wizard'), 'Brave [email protected]', 2008 ]
])
expand = ListExpanderExpand( static_source, 'namelist', 'name' ).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), len(list('El Agent')) + len(list('Serious Electron')) + len(list('Brave Wizard')) )
self.assertEqual( records[-1].getField('name').getValue(), 'd' )
self.assertEqual( records[-1].getField('email').getValue(), 'Brave [email protected]' )
def test_joinbykey_modifier(self):
source = StaticSource(
FieldSet([
Field('email', StringFieldType(), key=True),
Field('age', IntegerFieldType())
], FieldMap({
'email': 0,
'age': 1
})))
source.setResource(
[['El [email protected]', 12],
['Ochala [email protected]', 14],
['Sina [email protected]', 17],
['Akassa Savage [email protected]', 16],
['Sermak [email protected]', 22],
['Olivia Deadly [email protected]', 32],
['*****@*****.**', 42],
['Naria Cold-blodded [email protected]', 22],
['*****@*****.**', 54],
['Sina [email protected]', 56],
['Deadly [email protected]', 43],
['Mylenedriz [email protected]', 23],
['Calden [email protected]', 35],
['*****@*****.**', 56],
['Raven [email protected]', 23],
['*****@*****.**', 45],
['Pluto [email protected]', 64],
['Southern [email protected]', 53],
['Serious [email protected]', 62],
['*****@*****.**', 63],
['Risky [email protected]', 21],
['Rivatha [email protected]', 56],
['Panic [email protected]', 25],
['Tomara [email protected]', 46],
['Venessa [email protected]', 53],
['Western [email protected]', 71],
['*****@*****.**', 76]])
records = [
r for r in
JoinByKeyModifier(self.reader, fieldNames=['age'],
source=source).initialize().getRecords()
]
self.assertEqual(len(records), 85)
self.assertIsNone(records[-1].getField('age').getValue())
self.assertEqual(records[-2].getField('age').getValue(), 71)
self.assertEqual(records[0].getField('age').getValue(), 12)
def test_replace_words_by_source(self):
sourceRecords = [
FieldSet([
Field('from', StringFieldType(), defaultValue='killing'),
Field('to', StringFieldType(), defaultValue='born')
]),
FieldSet([
Field('from', StringFieldType(), defaultValue='sadness'),
Field('to', StringFieldType(), defaultValue='happyness')
]),
FieldSet([
Field('from', StringFieldType(), defaultValue='hungarian'),
Field('to', StringFieldType(), defaultValue='anywhere else')
])
]
self.field.setValue(
u'contains hungarian members attractive sadness killing')
self.field.setTransforms(
[ReplaceWordsBySourceTransform(sourceRecords, join='from')]).run()
self.assertEqual(
self.field.getValue(),
'contains anywhere else members attractive happyness born')
def test_field( self ):
static_source = StaticSource(
FieldSet([
Field( 'year', IntegerFieldType() ),
Field( 'country', StringFieldType() ),
Field( 'count', IntegerFieldType() ),
Field( 'cz', IntegerFieldType() ),
Field( 'hu', IntegerFieldType() ),
Field( 'sk', IntegerFieldType() ),
Field( 'pl', IntegerFieldType() )
],
FieldMap({
'year': 0,
'cz': 1,
'hu': 2,
'sk': 3,
'pl': 4
}))
)
static_source.setResource([
[1999,32,694,129,230],
[1999,395,392,297,453],
[1999,635,812,115,97]
])
expand = FieldExpand(
static_source,
fieldNamesAndLabels = {
'cz': 'Czech',
'hu': 'Hungary',
'sk': 'Slovak',
'pl': 'Poland',
},
valueFieldName = 'count',
labelFieldName = 'country'
).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 12 )
self.assertEqual( records[-1].getField('year').getValue(), 1999 )
self.assertEqual( records[-1].getField('count').getValue(), 812 )
self.assertEqual( records[-1].getField('pl').getValue(), 97 )
self.assertEqual( records[-1].getField('country').getValue(), 'Hungary' )
def test_append( self ):
source = CSVSource( FieldSet(
fields = [
Field( 'NUMBERS', StringFieldType() )
],
fieldmap = FieldMap({
'NUMBERS': 0
})
) )
source.setResource('tests/test_sources/test_csv_append.csv')
expand = AppendExpand( source, resource = 'tests/test_sources/test_csv_appended.csv' )
expand.initialize()
records = [ r for r in expand.getRecords() ]
expand.finalize()
self.assertEqual( len( records ), 6 )
self.assertEqual( records[0].getField('NUMBERS').getValue(), 'First' )
self.assertEqual( records[-1].getField('NUMBERS').getValue(), 'Sixth' )
def test_melt( self ):
static_source = StaticSource(
FieldSet([
Field( 'first', StringFieldType() ),
Field( 'height', FloatFieldType() ),
Field( 'last', StringFieldType() ),
Field( 'weight', IntegerFieldType() ),
Field( 'iq', IntegerFieldType() ),
Field( 'quantity', StringFieldType() ),
Field( 'value', FloatFieldType() )
],
FieldMap({
'first': 0,
'height': 1,
'last': 2,
'weight': 3,
'iq': 4
}))
)
static_source.setResource([
['John',5.5,'Doe',130,102],
['Mary',6.0,'Bo',150,98]
])
expand = MeltExpand(
static_source,
fieldNames = ['first','last'],
valueFieldName = 'value',
labelFieldName = 'quantity'
).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 6 )
self.assertEqual( len( records[0].getFieldNames() ), 4)
self.assertEqual( records[-1].getField('first').getValue(), 'Mary' )
self.assertEqual( records[-1].getField('last').getValue(), 'Bo' )
self.assertEqual( records[-1].getField('quantity').getValue(), 'height' )
self.assertEqual( records[-1].getField('value').getValue(), 6.0 )
def test_is_in_source_condition( self ):
false_source_records = [
FieldSet([
Field( 'integer', IntegerFieldType(), defaultValue = 1 ),
Field( 'float', FloatFieldType(), defaultValue = 4.2 ),
Field( 'date', DateFieldType(), defaultValue = '2012-02-04' )
]),
FieldSet([
Field( 'integer', IntegerFieldType(), defaultValue = 2 ),
Field( 'float', FloatFieldType(), defaultValue = 4.3 ),
Field( 'date', DateFieldType(), defaultValue = '2012-02-05' )
]),
FieldSet([
Field( 'integer', IntegerFieldType(), defaultValue = 3 ),
Field( 'float', FloatFieldType(), defaultValue = 4.4 ),
Field( 'date', DateFieldType(), defaultValue = '2012-02-06' )
])
]
self.assertFalse( IsInSourceCondition( false_source_records, 'integer' ).getResult( self.integer_field ) )
self.assertFalse( IsInSourceCondition( false_source_records, 'float' ).getResult( self.float_field ) )
self.assertFalse( IsInSourceCondition( false_source_records, 'date' ).getResult( self.date_field ) )
true_source_records = [
FieldSet([
Field( 'integer', IntegerFieldType(), defaultValue = 3 ),
Field( 'float', FloatFieldType(), defaultValue = 4.31 ),
Field( 'date', DateFieldType(), defaultValue = '2012-02-04' )
]),
FieldSet([
Field( 'integer', IntegerFieldType(), defaultValue = 4 ),
Field( 'float', FloatFieldType(), defaultValue = 4.32 ),
Field( 'date', DateFieldType(), defaultValue = '2013-02-05' )
]),
FieldSet([
Field( 'integer', IntegerFieldType(), defaultValue = 5 ),
Field( 'float', FloatFieldType(), defaultValue = 4.33 ),
Field( 'date', DateFieldType(), defaultValue = '2012-02-06' )
])
]
self.assertTrue( IsInSourceCondition( true_source_records, 'integer' ).getResult( self.integer_field ) )
self.assertTrue( IsInSourceCondition( true_source_records, 'float' ).getResult( self.float_field ) )
self.assertTrue( IsInSourceCondition( true_source_records, 'date' ).getResult( self.date_field ) )
def setUp(self):
self.reader = StaticSource(
FieldSet([
Field('name', StringFieldType()),
Field('email', StringFieldType()),
Field('year', IntegerFieldType()),
Field('after_year', IntegerFieldType()),
Field('number', IntegerFieldType())
], FieldMap({
'name': 0,
'email': 1,
'year': 2,
'after_year': 3
})))
self.reader.setResource(
[['El Agent', 'El [email protected]', 2008, 2008],
[
'Serious Electron', 'Serious [email protected]',
2008, 2013
],
['Brave Wizard', 'Brave [email protected]', 2008, 2008],
[
'Forgotten Itchy Emperor',
'Forgotten Itchy [email protected]', 2008, 2013
],
[
'The Moving Monkey', 'The Moving [email protected]',
2008, 2008
],
[
'Evil Ghostly Brigadier',
'Evil Ghostly [email protected]', 2008, 2013
],
[
'Strangely Oyster', 'Strangely [email protected]',
2008, 2008
],
[
'Anaconda Silver', 'Anaconda [email protected]', 2006,
2008
], ['Hawk Tough', 'Hawk [email protected]', 2004, 2008],
[
'The Disappointed Craw',
'The Disappointed [email protected]', 2008, 2013
], ['The Raven', 'The [email protected]', 1999, 2008],
[
'Ruby Boomerang', 'Ruby [email protected]', 2008,
2008
], ['Skunk Tough', 'Skunk [email protected]', 2010, 2008],
[
'The Nervous Forgotten Major',
'The Nervous Forgotten [email protected]', 2008, 2013
],
[
'Bursting Furious Puppet',
'Bursting Furious [email protected]', 2011, 2008
],
['Neptune Eagle', 'Neptune [email protected]', 2011, 2013],
['The Skunk', 'The [email protected]', 2008, 2013],
['Lone Demon', 'Lone [email protected]', 2008, 2008],
['The Skunk', 'The [email protected]', 1999, 2008],
[
'Gamma Serious Spear',
'Gamma Serious [email protected]', 2008, 2008
],
[
'Sleepy Dirty Sergeant',
'Sleepy Dirty [email protected]', 2008, 2008
], ['Red Monkey', 'Red [email protected]', 2008, 2008],
[
'Striking Tiger', 'Striking [email protected]', 2005,
2008
],
['Sliding Demon', 'Sliding [email protected]', 2011, 2008],
[
'Lone Commander', 'Lone [email protected]', 2008,
2013
],
['Dragon Insane', 'Dragon [email protected]', 2013, 2013],
['Demon Skilled', 'Demon [email protected]', 2011, 2004],
['Vulture Lucky', 'Vulture [email protected]', 2003, 2008],
['The Ranger', 'The [email protected]', 2013, 2008],
['Morbid Snake', 'Morbid [email protected]', 2011, 2008],
[
'Dancing Skeleton', 'Dancing [email protected]',
2001, 2004
], ['The Psycho', 'The [email protected]', 2005, 2008],
['Jupiter Rider', 'Jupiter [email protected]', 2011, 2008],
['Green Dog', 'Green [email protected]', 2011, 2008],
[
'Brutal Wild Colonel',
'Brutal Wild [email protected]', 2004, 2008
],
['Random Leader', 'Random [email protected]', 2008, 2008],
[
'Pluto Brigadier', 'Pluto [email protected]', 2008,
2004
],
[
'Southern Kangaroo', 'Southern [email protected]',
2008, 2008
],
['Serious Flea', 'Serious [email protected]', 2001, 2005],
[
'Nocturnal Raven', 'Nocturnal [email protected]', 2008,
2004
], ['Risky Flea', 'Risky [email protected]', 2005, 2005],
['The Corporal', 'The [email protected]', 2013, 2008],
[
'The Lucky Barbarian',
'The Lucky [email protected]', 2008, 2008
],
[
'Rocky Serious Dog', 'Rocky Serious [email protected]',
2008, 2008
],
[
'The Frozen Guardian',
'The Frozen [email protected]', 2008, 2008
],
[
'Freaky Frostbite', 'Freaky [email protected]',
2008, 2004
],
[
'The Tired Raven', 'The Tired [email protected]', 2008,
2008
],
[
'Disappointed Frostbite',
'Disappointed [email protected]', 2008, 2008
], ['The Craw', 'The [email protected]', 2003, 2008],
[
'Gutsy Strangely Chief',
'Gutsy Strangely [email protected]', 2008, 2008
], ['Queen Angry', 'Queen [email protected]', 2008, 2008],
[
'Pluto Albatross', 'Pluto [email protected]', 2003,
2008
],
[
'Endless Invader', 'Endless [email protected]', 2003,
2004
],
[
'Beta Young Sergeant',
'Beta Young [email protected]', 2008, 2011
], ['The Demon', 'The [email protected]', 2003, 2008],
['Lone Monkey', 'Lone [email protected]', 2011, 2008],
[
'Bursting Electron', 'Bursting [email protected]',
2003, 2010
],
[
'Gangster Solid', 'Gangster [email protected]', 2005,
2009
],
['The Gladiator', 'The [email protected]', 2001, 2002],
[
'Flash Frostbite', 'Flash [email protected]', 2005,
2004
],
[
'The Rainbow Pluto Demon',
'The Rainbow Pluto [email protected]', 2011, 2013
],
[
'Poseidon Rider', 'Poseidon [email protected]', 2008,
2006
],
[
'The Old Alpha Brigadier',
'The Old Alpha [email protected]', 2008, 2008
],
[
'Rough Anaconda', 'Rough [email protected]', 2001,
2011
],
[
'Tough Dinosaur', 'Tough [email protected]', 2011,
2010
],
[
'The Lost Dinosaur', 'The Lost [email protected]',
2008, 2008
], ['The Raven', 'The [email protected]', 2005, 2009],
['The Agent', 'The [email protected]', 2011, 2008],
[
'Brave Scarecrow', 'Brave [email protected]', 2008,
2007
],
[
'Flash Skeleton', 'Flash [email protected]', 2008,
2006
], ['The Admiral', 'The [email protected]', 1998, 2005],
['The Tombstone', 'The [email protected]', 2013, 2008],
['Golden Arrow', 'Golden [email protected]', 2008, 2005],
[
'White Guardian', 'White [email protected]', 2011,
2004
],
[
'The Black Eastern Power',
'The Black Eastern [email protected]', 2008, 2008
],
[
'Ruthless Soldier', 'Ruthless [email protected]',
2008, 2008
], ['Dirty Clown', 'Dirty [email protected]', 2008, 2008],
['Alpha Admiral', 'Alpha [email protected]', 2008, 2008],
[
'Lightning Major', 'Lightning [email protected]', 2008,
2008
],
[
'The Rock Demon', 'The Rock [email protected]', 2008,
2001
], ['Wild Tiger', 'Wild [email protected]', 2008, 2001],
[
'The Pointless Bandit',
'The Pointless [email protected]', 2008, 2008
],
['The Sergeant', 'The [email protected]', 1998, 2002],
['Western Ogre', 'Western [email protected]', 1998, 2004],
[
'Sergeant Strawberry',
'Sergeant [email protected]', 2006, 2008
]])
def test_set_modifier_source(self):
def setValue(record, field, scope):
return 'Found same email address' \
if record.getField('email').getValue() in [ sr.getField('email').getValue() for sr in scope.getSourceRecords() ] \
else 'Not found same email address'
fn = setValue
source = StaticSource(
FieldSet([
Field('name', StringFieldType()),
Field('email', StringFieldType())
], FieldMap({
'name': 0,
'email': 1
})))
source.setResource(
[['El Agent', 'El [email protected]'],
['Ochala Wild', 'Ochala [email protected]'],
['Sina Venomous', 'Sina [email protected]'],
[
'Akassa Savage Phalloz',
'Akassa Savage [email protected]'
], ['Sermak Bad', 'Sermak [email protected]'],
['Olivia Deadly Dawod', 'Olivia Deadly [email protected]'],
['Pendus Inhuman', '*****@*****.**'],
[
'Naria Cold-blodded Greste',
'Naria Cold-blodded [email protected]'
], ['Shard Brutal', '*****@*****.**'],
['Sina Cruel', 'Sina [email protected]'],
['Deadly Ohmar', 'Deadly [email protected]'],
[
'Mylenedriz Cold-blodded',
'Mylenedriz [email protected]'
], ['Calden Frigid', 'Calden [email protected]'],
['Acid Reaper', '*****@*****.**'],
['Raven Seth', 'Raven [email protected]'],
['Random Leader', '*****@*****.**'],
['Pluto Brigadier', 'Pluto [email protected]'],
['Southern Kangaroo', 'Southern [email protected]'],
['Serious Flea', 'Serious [email protected]'],
['Nocturnal Raven', '*****@*****.**'],
['Risky Flea', 'Risky [email protected]'],
['Rivatha Todal', 'Rivatha [email protected]'],
['Panic Oliviaezit', 'Panic [email protected]'],
['Tomara Wild', 'Tomara [email protected]'],
['Venessa Metalhead', 'Venessa [email protected]'],
['Western Ogre', 'Western [email protected]'],
['Sergeant Strawberry', '*****@*****.**']])
modifier = SetModifier(self.reader,
fieldNames=['name'],
source=source,
fn=fn)
modifier.initialize()
records = [r for r in modifier.getRecords()]
modifier.finalize()
self.assertEqual(len(records), 85)
self.assertEqual(
len([
r for r in records
if r.getField('name').getValue() == 'Found same email address'
]), 6)
self.assertEqual(
len([
r for r in records if r.getField('name').getValue() ==
'Not found same email address'
]), 79)
