def test_listexpander( self ):
static_source = StaticSource(
FieldSet([
Field( 'namelist', ListFieldType() ),
Field( 'email', StringFieldType() ),
Field( 'year', IntegerFieldType() ),
Field( 'name', StringFieldType() )
],
FieldMap({
'namelist': 0,
'email': 1,
'year': 2
}))
)
static_source.setResource([
[ list('El Agent'), 'El [email protected]', 2008 ],
[ list('Serious Electron'), 'Serious [email protected]', 2008 ],
[ list('Brave Wizard'), 'Brave [email protected]', 2008 ]
])
expand = ListExpanderExpand( static_source, 'namelist', 'name' ).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), len(list('El Agent')) + len(list('Serious Electron')) + len(list('Brave Wizard')) )
self.assertEqual( records[-1].getField('name').getValue(), 'd' )
self.assertEqual( records[-1].getField('email').getValue(), 'Brave [email protected]' )
def setUp( self ):
self.reader = StaticSource(
FieldSet([
Field( 'name', StringFieldType() ),
Field( 'email', StringFieldType() )
],
FieldMap({
'name': 0,
'email': 1
}))
)
self.reader.setResource([
[ 'Ochala Wild', 'Ochala [email protected]' ],
[ 'Sina Venomous', 'Sina [email protected]' ],
[ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ],
[ 'Sermak Bad', 'Sermak [email protected]' ],
[ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ],
[ 'Pendus Inhuman', 'Pendus [email protected]' ],
[ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ],
[ 'Shard Brutal', 'Shard [email protected]' ],
[ 'Sina Cruel', 'Sina [email protected]' ],
[ 'Deadly Ohmar', 'Deadly [email protected]' ],
[ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ],
[ 'Calden Frigid', 'Calden [email protected]' ],
[ 'Acid Reaper', 'Acid [email protected]' ],
[ 'Raven Seth', 'Raven [email protected]' ],
[ 'Rivatha Todal', 'Rivatha [email protected]' ],
[ 'Panic Oliviaezit', 'Panic [email protected]' ],
[ 'Tomara Wild', 'Tomara [email protected]' ],
[ 'Venessa Metalhead', 'Venessa [email protected]' ]
])
def test_listexpander_map( self ):
static_source = StaticSource(
FieldSet([
Field( 'city', StringFieldType() ),
Field( 'geometry', ListFieldType() ),
Field( 'latitude', FloatFieldType() ),
Field( 'longitude', FloatFieldType() )
],
FieldMap({
'city': 0,
'geometry': 1
}))
)
static_source.setResource([
[ 'Balatonlelle', [[17.6874552,46.7871465],[17.6865955,46.7870049],[17.6846158,46.7866786],[17.6834977,46.7864944],[17.6822251,46.7862847],[17.6815319,46.7861705],[17.6811473,46.7861071],[17.6795989,46.785852],[17.6774482,46.7854976],[17.6739061,46.7849139],[17.6729351,46.7847539],[17.6720789,46.7846318]] ],
[ 'Balatonlelle', [[17.6871206,46.7880197],[17.6874552,46.7871465]] ],
[ 'Balatonboglar', "[[17.6709959,46.7843474],[17.6710995,46.7840514],[17.6711443,46.7838791],[17.6711725,46.7837746],[17.6713146,46.7831483]]" ]
])
expand = ListExpanderExpand( static_source, 'geometry', expanderMap = { 'latitude': 1, 'longitude': 0 } ).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 19 )
self.assertEqual( records[10].getField('city').getValue(), 'Balatonlelle' )
self.assertEqual( records[10].getField('latitude').getValue(), 46.7847539 )
self.assertEqual( records[-1].getField('city').getValue(), 'Balatonboglar' )
self.assertEqual( records[-1].getField('longitude').getValue(), 17.6713146 )
def setUp( self ):
self.field = Field( 'name', StringFieldType() )
self.field.setTransforms([
IFStatement( IsEqualCondition('if_test') ),
SetTransform( 'if_test_valid' ),
ELIFStatement( IsEqualCondition('elif_test') ),
SetTransform( 'elif_test_valid' ),
ELSEStatement,
SetTransform( 'else_test_valid'),
ENDIFStatement,
ReturnTrueStatement
])
self.other_field = Field( 'name', StringFieldType() )
self.other_field.setTransforms([
IFNotStatement( IsEqualCondition('if_not_test') ),
IFNotStatement( IsEqualCondition('if_if_not_test') ),
SetTransform('if_if_not_test_valid'),
ELIFNotStatement( IsEqualCondition('if_elif_not_test') ),
SetTransform('if_elif_not_test_valid'),
ENDIFStatement,
ENDIFStatement,
ReturnTrueStatement
])
def test_is_match_by_regexp_condition( self ):
match_field = Field( 'string', StringFieldType(), defaultValue = 'It will match, because it has 1 or more number value.' )
not_match_field = Field( 'string', StringFieldType(), defaultValue = 'It will match, because it has one or more number value.' )
self.assertTrue( IsMatchByRegexpCondition( r'^.*(\d+).*$' ).getResult( match_field ) )
self.assertFalse( IsMatchByRegexpCondition( r'^.*(\d+).*$' ).getResult( not_match_field ) )
self.assertFalse( IsMatchByRegexpCondition( r'^.*(\d+).*$' ).getResult( self.empty_str_field ) )
def test_joinbykey_modifier(self):
source = StaticSource(
FieldSet([
Field('email', StringFieldType(), key=True),
Field('age', IntegerFieldType())
], FieldMap({
'email': 0,
'age': 1
})))
source.setResource(
[['El [email protected]', 12],
['Ochala [email protected]', 14],
['Sina [email protected]', 17],
['Akassa Savage [email protected]', 16],
['Sermak [email protected]', 22],
['Olivia Deadly [email protected]', 32],
['*****@*****.**', 42],
['Naria Cold-blodded [email protected]', 22],
['*****@*****.**', 54],
['Sina [email protected]', 56],
['Deadly [email protected]', 43],
['Mylenedriz [email protected]', 23],
['Calden [email protected]', 35],
['*****@*****.**', 56],
['Raven [email protected]', 23],
['*****@*****.**', 45],
['Pluto [email protected]', 64],
['Southern [email protected]', 53],
['Serious [email protected]', 62],
['*****@*****.**', 63],
['Risky [email protected]', 21],
['Rivatha [email protected]', 56],
['Panic [email protected]', 25],
['Tomara [email protected]', 46],
['Venessa [email protected]', 53],
['Western [email protected]', 71],
['*****@*****.**', 76]])
records = [
r for r in
JoinByKeyModifier(self.reader, fieldNames=['age'],
source=source).initialize().getRecords()
]
self.assertEqual(len(records), 85)
self.assertIsNone(records[-1].getField('age').getValue())
self.assertEqual(records[-2].getField('age').getValue(), 71)
self.assertEqual(records[0].getField('age').getValue(), 12)
def test_remove_words_by_source(self):
sourceRecords = [
FieldSet(
[Field('name', StringFieldType(), defaultValue='killing')]),
FieldSet(
[Field('name', StringFieldType(), defaultValue='sadness')]),
FieldSet(
[Field('name', StringFieldType(), defaultValue='hungarian')])
]
self.field.setValue(
u'contains hungarian members attractive sadness killing')
self.field.setTransforms([RemoveWordsBySourceTransform(sourceRecords)
]).run()
self.assertEqual(self.field.getValue(), 'contains members attractive')
def setUp(self):
self.fs = FieldSet([
Field('id', IntegerFieldType(), defaultValue=1, key=True),
Field('name', StringFieldType(), defaultValue='John doe'),
Field('email', StringFieldType(), key=True),
Field('birth_date',
DateFieldType(),
field_final_type=StringFieldType()),
Field('favourite_number', FloatFieldType()),
Field('created',
DateTimeFieldType(),
defaultValue='2013-04-04 18:44:32'),
Field('updated',
DateTimeFieldType(),
defaultValue='2013-04-04 18:44:32')
])
self.fm = FieldMap({
'id': 0,
'name': 1,
'email': 2,
'birth_date': 3,
'favourite_number': 4,
'created': 5,
'updated': 6,
'not_existing_field_name': 6
})
def setUp(self):
self.field = Field('name', StringFieldType())
self.field.setTransforms([
IFStatement(IsEqualCondition('if_test')),
SetTransform('if_test_valid'),
ELIFStatement(IsEqualCondition('elif_test')),
SetTransform('elif_test_valid'), ELSEStatement,
SetTransform('else_test_valid'), ENDIFStatement,
ReturnTrueStatement
])
self.other_field = Field('name', StringFieldType())
self.other_field.setTransforms([
IFNotStatement(IsEqualCondition('if_not_test')),
IFNotStatement(IsEqualCondition('if_if_not_test')),
SetTransform('if_if_not_test_valid'),
ELIFNotStatement(IsEqualCondition('if_elif_not_test')),
SetTransform('if_elif_not_test_valid'), ENDIFStatement,
ENDIFStatement, ReturnTrueStatement
])
def test_field_manipulation(self):
with self.assertRaises(FieldNotExistsError):
self.fs.deleteField('key')
with self.assertRaises(TypeError):
self.fs.addField('key')
self.assertEqual(len(self.fs.getFields()), 7)
self.fs.deleteField('birth_date')
self.assertEqual(len(self.fs.getFields()), 6)
self.fs.addField(Field('language', StringFieldType()))
self.assertEqual(len(self.fs.getFields()), 7)
def setUp( self ):
self.integer_field = Field( 'integer', IntegerFieldType(), defaultValue = 4 )
self.float_field = Field( 'float', FloatFieldType(), defaultValue = 4.32 )
self.date_field = Field( 'date', DateFieldType(), defaultValue = '2013-02-05' )
self.string_field = Field( 'string', StringFieldType(), defaultValue = 'test' )
self.empty_int_field = Field( 'integer_empty', IntegerFieldType() )
self.empty_str_field = Field( 'string_empty', StringFieldType(), defaultValue = '' )
def test_append( self ):
source = CSVSource( FieldSet(
fields = [
Field( 'NUMBERS', StringFieldType() )
],
fieldmap = FieldMap({
'NUMBERS': 0
})
) )
source.setResource('tests/test_sources/test_csv_append.csv')
expand = AppendExpand( source, resource = 'tests/test_sources/test_csv_appended.csv' )
expand.initialize()
records = [ r for r in expand.getRecords() ]
expand.finalize()
self.assertEqual( len( records ), 6 )
self.assertEqual( records[0].getField('NUMBERS').getValue(), 'First' )
self.assertEqual( records[-1].getField('NUMBERS').getValue(), 'Sixth' )
def test_field( self ):
static_source = StaticSource(
FieldSet([
Field( 'year', IntegerFieldType() ),
Field( 'country', StringFieldType() ),
Field( 'count', IntegerFieldType() ),
Field( 'cz', IntegerFieldType() ),
Field( 'hu', IntegerFieldType() ),
Field( 'sk', IntegerFieldType() ),
Field( 'pl', IntegerFieldType() )
],
FieldMap({
'year': 0,
'cz': 1,
'hu': 2,
'sk': 3,
'pl': 4
}))
)
static_source.setResource([
[1999,32,694,129,230],
[1999,395,392,297,453],
[1999,635,812,115,97]
])
expand = FieldExpand(
static_source,
fieldNamesAndLabels = {
'cz': 'Czech',
'hu': 'Hungary',
'sk': 'Slovak',
'pl': 'Poland',
},
valueFieldName = 'count',
labelFieldName = 'country'
).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 12 )
self.assertEqual( records[-1].getField('year').getValue(), 1999 )
self.assertEqual( records[-1].getField('count').getValue(), 812 )
self.assertEqual( records[-1].getField('pl').getValue(), 97 )
self.assertEqual( records[-1].getField('country').getValue(), 'Hungary' )
def test_melt( self ):
static_source = StaticSource(
FieldSet([
Field( 'first', StringFieldType() ),
Field( 'height', FloatFieldType() ),
Field( 'last', StringFieldType() ),
Field( 'weight', IntegerFieldType() ),
Field( 'iq', IntegerFieldType() ),
Field( 'quantity', StringFieldType() ),
Field( 'value', FloatFieldType() )
],
FieldMap({
'first': 0,
'height': 1,
'last': 2,
'weight': 3,
'iq': 4
}))
)
static_source.setResource([
['John',5.5,'Doe',130,102],
['Mary',6.0,'Bo',150,98]
])
expand = MeltExpand(
static_source,
fieldNames = ['first','last'],
valueFieldName = 'value',
labelFieldName = 'quantity'
).initialize()
records = [ r for r in expand.getRecords() ]
self.assertEqual( len( records ), 6 )
self.assertEqual( len( records[0].getFieldNames() ), 4)
self.assertEqual( records[-1].getField('first').getValue(), 'Mary' )
self.assertEqual( records[-1].getField('last').getValue(), 'Bo' )
self.assertEqual( records[-1].getField('quantity').getValue(), 'height' )
self.assertEqual( records[-1].getField('value').getValue(), 6.0 )
def test_replace_words_by_source(self):
sourceRecords = [
FieldSet([
Field('from', StringFieldType(), defaultValue='killing'),
Field('to', StringFieldType(), defaultValue='born')
]),
FieldSet([
Field('from', StringFieldType(), defaultValue='sadness'),
Field('to', StringFieldType(), defaultValue='happyness')
]),
FieldSet([
Field('from', StringFieldType(), defaultValue='hungarian'),
Field('to', StringFieldType(), defaultValue='anywhere else')
])
]
self.field.setValue(
u'contains hungarian members attractive sadness killing')
self.field.setTransforms(
[ReplaceWordsBySourceTransform(sourceRecords, join='from')]).run()
self.assertEqual(
self.field.getValue(),
'contains anywhere else members attractive happyness born')
def setUp(self):
self.reader = StaticSource(
FieldSet([
Field('name', StringFieldType()),
Field('email', StringFieldType()),
Field('year', IntegerFieldType()),
Field('after_year', IntegerFieldType()),
Field('number', IntegerFieldType())
], FieldMap({
'name': 0,
'email': 1,
'year': 2,
'after_year': 3
})))
self.reader.setResource(
[['El Agent', 'El [email protected]', 2008, 2008],
[
'Serious Electron', 'Serious [email protected]',
2008, 2013
],
['Brave Wizard', 'Brave [email protected]', 2008, 2008],
[
'Forgotten Itchy Emperor',
'Forgotten Itchy [email protected]', 2008, 2013
],
[
'The Moving Monkey', 'The Moving [email protected]',
2008, 2008
],
[
'Evil Ghostly Brigadier',
'Evil Ghostly [email protected]', 2008, 2013
],
[
'Strangely Oyster', 'Strangely [email protected]',
2008, 2008
],
[
'Anaconda Silver', 'Anaconda [email protected]', 2006,
2008
], ['Hawk Tough', 'Hawk [email protected]', 2004, 2008],
[
'The Disappointed Craw',
'The Disappointed [email protected]', 2008, 2013
], ['The Raven', 'The [email protected]', 1999, 2008],
[
'Ruby Boomerang', 'Ruby [email protected]', 2008,
2008
], ['Skunk Tough', 'Skunk [email protected]', 2010, 2008],
[
'The Nervous Forgotten Major',
'The Nervous Forgotten [email protected]', 2008, 2013
],
[
'Bursting Furious Puppet',
'Bursting Furious [email protected]', 2011, 2008
],
['Neptune Eagle', 'Neptune [email protected]', 2011, 2013],
['The Skunk', 'The [email protected]', 2008, 2013],
['Lone Demon', 'Lone [email protected]', 2008, 2008],
['The Skunk', 'The [email protected]', 1999, 2008],
[
'Gamma Serious Spear',
'Gamma Serious [email protected]', 2008, 2008
],
[
'Sleepy Dirty Sergeant',
'Sleepy Dirty [email protected]', 2008, 2008
], ['Red Monkey', 'Red [email protected]', 2008, 2008],
[
'Striking Tiger', 'Striking [email protected]', 2005,
2008
],
['Sliding Demon', 'Sliding [email protected]', 2011, 2008],
[
'Lone Commander', 'Lone [email protected]', 2008,
2013
],
['Dragon Insane', 'Dragon [email protected]', 2013, 2013],
['Demon Skilled', 'Demon [email protected]', 2011, 2004],
['Vulture Lucky', 'Vulture [email protected]', 2003, 2008],
['The Ranger', 'The [email protected]', 2013, 2008],
['Morbid Snake', 'Morbid [email protected]', 2011, 2008],
[
'Dancing Skeleton', 'Dancing [email protected]',
2001, 2004
], ['The Psycho', 'The [email protected]', 2005, 2008],
['Jupiter Rider', 'Jupiter [email protected]', 2011, 2008],
['Green Dog', 'Green [email protected]', 2011, 2008],
[
'Brutal Wild Colonel',
'Brutal Wild [email protected]', 2004, 2008
],
['Random Leader', 'Random [email protected]', 2008, 2008],
[
'Pluto Brigadier', 'Pluto [email protected]', 2008,
2004
],
[
'Southern Kangaroo', 'Southern [email protected]',
2008, 2008
],
['Serious Flea', 'Serious [email protected]', 2001, 2005],
[
'Nocturnal Raven', 'Nocturnal [email protected]', 2008,
2004
], ['Risky Flea', 'Risky [email protected]', 2005, 2005],
['The Corporal', 'The [email protected]', 2013, 2008],
[
'The Lucky Barbarian',
'The Lucky [email protected]', 2008, 2008
],
[
'Rocky Serious Dog', 'Rocky Serious [email protected]',
2008, 2008
],
[
'The Frozen Guardian',
'The Frozen [email protected]', 2008, 2008
],
[
'Freaky Frostbite', 'Freaky [email protected]',
2008, 2004
],
[
'The Tired Raven', 'The Tired [email protected]', 2008,
2008
],
[
'Disappointed Frostbite',
'Disappointed [email protected]', 2008, 2008
], ['The Craw', 'The [email protected]', 2003, 2008],
[
'Gutsy Strangely Chief',
'Gutsy Strangely [email protected]', 2008, 2008
], ['Queen Angry', 'Queen [email protected]', 2008, 2008],
[
'Pluto Albatross', 'Pluto [email protected]', 2003,
2008
],
[
'Endless Invader', 'Endless [email protected]', 2003,
2004
],
[
'Beta Young Sergeant',
'Beta Young [email protected]', 2008, 2011
], ['The Demon', 'The [email protected]', 2003, 2008],
['Lone Monkey', 'Lone [email protected]', 2011, 2008],
[
'Bursting Electron', 'Bursting [email protected]',
2003, 2010
],
[
'Gangster Solid', 'Gangster [email protected]', 2005,
2009
],
['The Gladiator', 'The [email protected]', 2001, 2002],
[
'Flash Frostbite', 'Flash [email protected]', 2005,
2004
],
[
'The Rainbow Pluto Demon',
'The Rainbow Pluto [email protected]', 2011, 2013
],
[
'Poseidon Rider', 'Poseidon [email protected]', 2008,
2006
],
[
'The Old Alpha Brigadier',
'The Old Alpha [email protected]', 2008, 2008
],
[
'Rough Anaconda', 'Rough [email protected]', 2001,
2011
],
[
'Tough Dinosaur', 'Tough [email protected]', 2011,
2010
],
[
'The Lost Dinosaur', 'The Lost [email protected]',
2008, 2008
], ['The Raven', 'The [email protected]', 2005, 2009],
['The Agent', 'The [email protected]', 2011, 2008],
[
'Brave Scarecrow', 'Brave [email protected]', 2008,
2007
],
[
'Flash Skeleton', 'Flash [email protected]', 2008,
2006
], ['The Admiral', 'The [email protected]', 1998, 2005],
['The Tombstone', 'The [email protected]', 2013, 2008],
['Golden Arrow', 'Golden [email protected]', 2008, 2005],
[
'White Guardian', 'White [email protected]', 2011,
2004
],
[
'The Black Eastern Power',
'The Black Eastern [email protected]', 2008, 2008
],
[
'Ruthless Soldier', 'Ruthless [email protected]',
2008, 2008
], ['Dirty Clown', 'Dirty [email protected]', 2008, 2008],
['Alpha Admiral', 'Alpha [email protected]', 2008, 2008],
[
'Lightning Major', 'Lightning [email protected]', 2008,
2008
],
[
'The Rock Demon', 'The Rock [email protected]', 2008,
2001
], ['Wild Tiger', 'Wild [email protected]', 2008, 2001],
[
'The Pointless Bandit',
'The Pointless [email protected]', 2008, 2008
],
['The Sergeant', 'The [email protected]', 1998, 2002],
['Western Ogre', 'Western [email protected]', 1998, 2004],
[
'Sergeant Strawberry',
'Sergeant [email protected]', 2006, 2008
]])
class Test_Statement( unittest.TestCase ):
def setUp( self ):
self.field = Field( 'name', StringFieldType() )
self.field.setTransforms([
IFStatement( IsEqualCondition('if_test') ),
SetTransform( 'if_test_valid' ),
ELIFStatement( IsEqualCondition('elif_test') ),
SetTransform( 'elif_test_valid' ),
ELSEStatement,
SetTransform( 'else_test_valid'),
ENDIFStatement,
ReturnTrueStatement
])
self.other_field = Field( 'name', StringFieldType() )
self.other_field.setTransforms([
IFNotStatement( IsEqualCondition('if_not_test') ),
IFNotStatement( IsEqualCondition('if_if_not_test') ),
SetTransform('if_if_not_test_valid'),
ELIFNotStatement( IsEqualCondition('if_elif_not_test') ),
SetTransform('if_elif_not_test_valid'),
ENDIFStatement,
ENDIFStatement,
ReturnTrueStatement
])
def test_if( self ):
self.field.setValue('if_test')
self.field.run()
self.assertEqual( self.field.getValue(), 'if_test_valid' )
def test_elif( self ):
self.field.setValue('elif_test')
self.field.run()
self.assertEqual( self.field.getValue(), 'elif_test_valid' )
def test_else( self ):
self.field.setValue('else_test')
self.field.run()
self.assertEqual( self.field.getValue(), 'else_test_valid' )
def test_if_not( self ):
self.other_field.setValue('if_if_not_test')
self.other_field.run()
self.assertEqual( self.other_field.getValue(), 'if_elif_not_test_valid' )
def test_elif_not( self ):
self.other_field.setValue('if_elif_not_test')
self.other_field.run()
self.assertEqual( self.other_field.getValue(), 'if_if_not_test_valid' )
def setUp(self):
self.field = Field('string', StringFieldType())
class Test_Transform(unittest.TestCase):
def setUp(self):
self.field = Field('string', StringFieldType())
def test_clean_transform(self):
self.field.setValue(
' That is a good sentence, which is contains many english word! ')
self.field.setTransforms([CleanTransform(replaces={'many': '1+'})
]).run()
self.assertEqual(
self.field.getValue(),
'That is a good sentence which is contains 1+ english word')
def test_converttype_transform_soft(self):
self.field.setValue('2012-02-12')
self.field.setFinalType(DateFieldType())
self.field.setTransforms([ConvertTypeTransform('Date')]).run()
self.assertEqual(self.field.getValue(), datetime.date(2012, 2, 12))
def test_converttype_transform_soft_invalid(self):
self.field.setValue('four miles')
self.field.setFinalType(DateFieldType())
with self.assertRaises(FieldTypeError):
self.field.setTransforms([ConvertTypeTransform('Date')]).run()
def test_converttype_transform_hard(self):
self.field.setValue('four miles')
self.field.setFinalType(DateFieldType())
self.field.setTransforms([ConvertTypeTransform('Date',
hard=True)]).run()
self.assertIsNone(self.field.getValue())
def test_converttype_transform_hard_with_default(self):
self.field.setValue('four miles')
self.field.setFinalType(DateFieldType())
self.field.setTransforms([
ConvertTypeTransform('Date',
hard=True,
defaultValue=datetime.date.today())
]).run()
self.assertEqual(self.field.getValue(), datetime.date.today())
def test_homogenize_transform(self):
self.field.setValue(u'árvíztűrőtükörfúrógépÁRVÍZTŰRŐTÜKÖRFÚRÓGÉP')
self.field.setTransforms([HomogenizeTransform]).run()
self.assertEqual(self.field.getValue(),
'arvizturotukorfurogeparvizturotukorfurogep')
def test_lowercase_transform(self):
self.field.setValue(
u'That is a good sentence, which is contains many english word!')
self.field.setTransforms([LowerCaseTransform]).run()
self.assertEqual(
self.field.getValue(),
'that is a good sentence, which is contains many english word!')
def test_uppercase_transform(self):
self.field.setValue(
u'That is a good sentence, which is contains many english word!')
self.field.setTransforms([UpperCaseTransform]).run()
self.assertEqual(
self.field.getValue(),
'THAT IS A GOOD SENTENCE, WHICH IS CONTAINS MANY ENGLISH WORD!')
def test_strip_transform(self):
self.field.setValue(
u' That is a good sentence, which is contains many english word! '
)
self.field.setTransforms([StripTransform()]).run()
self.assertEqual(
self.field.getValue(),
'That is a good sentence, which is contains many english word!')
def test_title_transform(self):
self.field.setValue(
u'That is a good sentence, which is contains many english word!')
self.field.setTransforms([TitleTransform]).run()
self.assertEqual(
self.field.getValue(),
'That Is A Good Sentence, Which Is Contains Many English Word!')
def test_stem_transform(self):
self.field.setValue(
u'contains hungarian members attractive sadness killing')
self.field.setTransforms([StemTransform('English')]).run()
self.assertEqual(self.field.getValue(),
'contain hungarian member attract sad kill')
def test_set_transform(self):
self.field.setTransforms(
[SetTransform('That will be the final value')]).run()
self.assertEqual(self.field.getValue(), 'That will be the final value')
def test_set_transform_srintfs(self):
self.field.setValue('Myself')
self.field.setTransforms([SetTransform('%(self)s or the new string')
]).run()
self.assertEqual(self.field.getValue(), 'Myself or the new string')
def test_map_transform(self):
self.field.setValue(u'Monkey')
self.field.setTransforms([MapTransform({'monkey': 'animal'})]).run()
self.assertEqual(self.field.getValue(), 'Monkey')
def test_map_transform_ignore_case(self):
self.field.setValue(u'Monkey')
self.field.setTransforms(
[MapTransform({'monkey': 'animal'}, ignorecase=True)]).run()
self.assertEqual(self.field.getValue(), 'animal')
def test_map_transform_not_used_else_value(self):
self.field.setValue(u'monkey')
self.field.setTransforms(
[MapTransform({'monkey': 'animal'}, elseValue='other')]).run()
self.assertEqual(self.field.getValue(), 'animal')
def test_map_transform_used_else_value(self):
self.field.setValue(u'Building')
self.field.setTransforms(
[MapTransform({'monkey': 'animal'}, elseValue='other')]).run()
self.assertEqual(self.field.getValue(), 'other')
def test_map_transform_used_else_clear(self):
self.field.setValue(u'Building')
self.field.setTransforms(
[MapTransform({'monkey': 'animal'}, elseClear=True)]).run()
self.assertIsNone(self.field.getValue())
def test_replace_by_regexp(self):
self.field.setValue(u'2012-04-20')
self.field.setTransforms([
ReplaceByRegexpTransform(regexp='^([0-9]{4}-[0-9]{2})-[0-9]{2}$',
to='$1')
]).run()
self.assertEqual(self.field.getValue(), '2012-04')
def test_remove_words_by_source(self):
sourceRecords = [
FieldSet(
[Field('name', StringFieldType(), defaultValue='killing')]),
FieldSet(
[Field('name', StringFieldType(), defaultValue='sadness')]),
FieldSet(
[Field('name', StringFieldType(), defaultValue='hungarian')])
]
self.field.setValue(
u'contains hungarian members attractive sadness killing')
self.field.setTransforms([RemoveWordsBySourceTransform(sourceRecords)
]).run()
self.assertEqual(self.field.getValue(), 'contains members attractive')
def test_replace_words_by_source(self):
sourceRecords = [
FieldSet([
Field('from', StringFieldType(), defaultValue='killing'),
Field('to', StringFieldType(), defaultValue='born')
]),
FieldSet([
Field('from', StringFieldType(), defaultValue='sadness'),
Field('to', StringFieldType(), defaultValue='happyness')
]),
FieldSet([
Field('from', StringFieldType(), defaultValue='hungarian'),
Field('to', StringFieldType(), defaultValue='anywhere else')
])
]
self.field.setValue(
u'contains hungarian members attractive sadness killing')
self.field.setTransforms(
[ReplaceWordsBySourceTransform(sourceRecords, join='from')]).run()
self.assertEqual(
self.field.getValue(),
'contains anywhere else members attractive happyness born')
def test_split_transform_first(self):
self.field.setValue(
u'contains hungarian members attractive sadness killing')
self.field.setTransforms([SplitTransform('0')]).run()
self.assertEqual(self.field.getValue(), 'contains')
def test_split_transform_last(self):
self.field.setValue(
u'contains hungarian members attractive sadness killing')
self.field.setTransforms([SplitTransform('-1')]).run()
self.assertEqual(self.field.getValue(), 'killing')
def test_split_transform_interval(self):
self.field.setValue(
u'contains hungarian members attractive sadness killing')
self.field.setTransforms([SplitTransform('1:-1')]).run()
self.assertEqual(self.field.getValue(),
'hungarian members attractive sadness')
def setUp( self ):
self.field = Field( 'string', StringFieldType() )
class Test_Transform( unittest.TestCase ):
def setUp( self ):
self.field = Field( 'string', StringFieldType() )
def test_clean_transform( self ):
self.field.setValue(' That is a good sentence, which is contains many english word! ')
self.field.setTransforms([ CleanTransform( replaces = { 'many': '1+' } ) ]).run()
self.assertEqual( self.field.getValue(), 'That is a good sentence which is contains 1+ english word' )
def test_converttype_transform_soft( self ):
self.field.setValue('2012-02-12')
self.field.setFinalType( DateFieldType() )
self.field.setTransforms([ ConvertTypeTransform( 'Date' ) ]).run()
self.assertEqual( self.field.getValue(), datetime.date( 2012, 2, 12 ) )
def test_converttype_transform_soft_invalid( self ):
self.field.setValue('four miles')
self.field.setFinalType( DateFieldType() )
with self.assertRaises( FieldTypeError ):
self.field.setTransforms([ ConvertTypeTransform( 'Date' ) ]).run()
def test_converttype_transform_hard( self ):
self.field.setValue('four miles')
self.field.setFinalType( DateFieldType() )
self.field.setTransforms([ ConvertTypeTransform( 'Date', hard = True ) ]).run()
self.assertIsNone( self.field.getValue() )
def test_converttype_transform_hard_with_default( self ):
self.field.setValue('four miles')
self.field.setFinalType( DateFieldType() )
self.field.setTransforms([ ConvertTypeTransform( 'Date', hard = True, defaultValue = datetime.date.today() ) ]).run()
self.assertEqual( self.field.getValue(), datetime.date.today() )
def test_homogenize_transform( self ):
self.field.setValue(u'árvíztűrőtükörfúrógépÁRVÍZTŰRŐTÜKÖRFÚRÓGÉP')
self.field.setTransforms([ HomogenizeTransform ]).run()
self.assertEqual( self.field.getValue(), 'arvizturotukorfurogeparvizturotukorfurogep' )
def test_lowercase_transform( self ):
self.field.setValue(u'That is a good sentence, which is contains many english word!')
self.field.setTransforms([ LowerCaseTransform ]).run()
self.assertEqual( self.field.getValue(), 'that is a good sentence, which is contains many english word!' )
def test_uppercase_transform( self ):
self.field.setValue(u'That is a good sentence, which is contains many english word!')
self.field.setTransforms([ UpperCaseTransform ]).run()
self.assertEqual( self.field.getValue(), 'THAT IS A GOOD SENTENCE, WHICH IS CONTAINS MANY ENGLISH WORD!' )
def test_strip_transform( self ):
self.field.setValue(u' That is a good sentence, which is contains many english word! ')
self.field.setTransforms([ StripTransform() ]).run()
self.assertEqual( self.field.getValue(), 'That is a good sentence, which is contains many english word!' )
def test_title_transform( self ):
self.field.setValue(u'That is a good sentence, which is contains many english word!')
self.field.setTransforms([ TitleTransform ]).run()
self.assertEqual( self.field.getValue(), 'That Is A Good Sentence, Which Is Contains Many English Word!' )
def test_stem_transform( self ):
self.field.setValue(u'contains hungarian members attractive sadness killing')
self.field.setTransforms([ StemTransform( 'English' ) ]).run()
self.assertEqual( self.field.getValue(), 'contain hungarian member attract sad kill' )
def test_set_transform( self ):
self.field.setTransforms([ SetTransform( 'That will be the final value' ) ]).run()
self.assertEqual( self.field.getValue(), 'That will be the final value' )
def test_set_transform_srintfs( self ):
self.field.setValue('Myself')
self.field.setTransforms([ SetTransform( '%(self)s or the new string' ) ]).run()
self.assertEqual( self.field.getValue(), 'Myself or the new string' )
def test_map_transform( self ):
self.field.setValue(u'Monkey')
self.field.setTransforms([ MapTransform({ 'monkey': 'animal' }) ]).run()
self.assertEqual( self.field.getValue(), 'Monkey' )
def test_map_transform_ignore_case( self ):
self.field.setValue(u'Monkey')
self.field.setTransforms([ MapTransform({ 'monkey': 'animal' }, ignorecase = True ) ]).run()
self.assertEqual( self.field.getValue(), 'animal' )
def test_map_transform_not_used_else_value( self ):
self.field.setValue(u'monkey')
self.field.setTransforms([ MapTransform({ 'monkey': 'animal' }, elseValue = 'other' ) ]).run()
self.assertEqual( self.field.getValue(), 'animal' )
def test_map_transform_used_else_value( self ):
self.field.setValue(u'Building')
self.field.setTransforms([ MapTransform({ 'monkey': 'animal' }, elseValue = 'other' ) ]).run()
self.assertEqual( self.field.getValue(), 'other' )
def test_map_transform_used_else_clear( self ):
self.field.setValue(u'Building')
self.field.setTransforms([ MapTransform({ 'monkey': 'animal' }, elseClear = True ) ]).run()
self.assertIsNone( self.field.getValue() )
def test_replace_by_regexp( self ):
self.field.setValue(u'2012-04-20')
self.field.setTransforms([ ReplaceByRegexpTransform( regexp = '^([0-9]{4}-[0-9]{2})-[0-9]{2}$', to = '$1' ) ]).run()
self.assertEqual( self.field.getValue(), '2012-04' )
def test_remove_words_by_source( self ):
sourceRecords = [
FieldSet([ Field( 'name', StringFieldType(), defaultValue = 'killing' ) ]),
FieldSet([ Field( 'name', StringFieldType(), defaultValue = 'sadness' ) ]),
FieldSet([ Field( 'name', StringFieldType(), defaultValue = 'hungarian' ) ])
]
self.field.setValue(u'contains hungarian members attractive sadness killing')
self.field.setTransforms([ RemoveWordsBySourceTransform( sourceRecords ) ]).run()
self.assertEqual( self.field.getValue(), 'contains members attractive' )
def test_replace_words_by_source( self ):
sourceRecords = [
FieldSet([
Field( 'from', StringFieldType(), defaultValue = 'killing' ),
Field( 'to', StringFieldType(), defaultValue = 'born' )
]),
FieldSet([
Field( 'from', StringFieldType(), defaultValue = 'sadness' ),
Field( 'to', StringFieldType(), defaultValue = 'happyness' )
]),
FieldSet([
Field( 'from', StringFieldType(), defaultValue = 'hungarian' ),
Field( 'to', StringFieldType(), defaultValue = 'anywhere else' )
])
]
self.field.setValue(u'contains hungarian members attractive sadness killing')
self.field.setTransforms([ ReplaceWordsBySourceTransform( sourceRecords, join = 'from' ) ]).run()
self.assertEqual( self.field.getValue(), 'contains anywhere else members attractive happyness born' )
def test_split_transform_first( self ):
self.field.setValue(u'contains hungarian members attractive sadness killing')
self.field.setTransforms([ SplitTransform( '0' ) ]).run()
self.assertEqual( self.field.getValue(), 'contains' )
def test_split_transform_last( self ):
self.field.setValue(u'contains hungarian members attractive sadness killing')
self.field.setTransforms([ SplitTransform( '-1' ) ]).run()
self.assertEqual( self.field.getValue(), 'killing' )
def test_split_transform_interval( self ):
self.field.setValue(u'contains hungarian members attractive sadness killing')
self.field.setTransforms([ SplitTransform( '1:-1' ) ]).run()
self.assertEqual( self.field.getValue(), 'hungarian members attractive sadness' )
def test_set_modifier_source(self):
def setValue(record, field, scope):
return 'Found same email address' \
if record.getField('email').getValue() in [ sr.getField('email').getValue() for sr in scope.getSourceRecords() ] \
else 'Not found same email address'
fn = setValue
source = StaticSource(
FieldSet([
Field('name', StringFieldType()),
Field('email', StringFieldType())
], FieldMap({
'name': 0,
'email': 1
})))
source.setResource(
[['El Agent', 'El [email protected]'],
['Ochala Wild', 'Ochala [email protected]'],
['Sina Venomous', 'Sina [email protected]'],
[
'Akassa Savage Phalloz',
'Akassa Savage [email protected]'
], ['Sermak Bad', 'Sermak [email protected]'],
['Olivia Deadly Dawod', 'Olivia Deadly [email protected]'],
['Pendus Inhuman', '*****@*****.**'],
[
'Naria Cold-blodded Greste',
'Naria Cold-blodded [email protected]'
], ['Shard Brutal', '*****@*****.**'],
['Sina Cruel', 'Sina [email protected]'],
['Deadly Ohmar', 'Deadly [email protected]'],
[
'Mylenedriz Cold-blodded',
'Mylenedriz [email protected]'
], ['Calden Frigid', 'Calden [email protected]'],
['Acid Reaper', '*****@*****.**'],
['Raven Seth', 'Raven [email protected]'],
['Random Leader', '*****@*****.**'],
['Pluto Brigadier', 'Pluto [email protected]'],
['Southern Kangaroo', 'Southern [email protected]'],
['Serious Flea', 'Serious [email protected]'],
['Nocturnal Raven', '*****@*****.**'],
['Risky Flea', 'Risky [email protected]'],
['Rivatha Todal', 'Rivatha [email protected]'],
['Panic Oliviaezit', 'Panic [email protected]'],
['Tomara Wild', 'Tomara [email protected]'],
['Venessa Metalhead', 'Venessa [email protected]'],
['Western Ogre', 'Western [email protected]'],
['Sergeant Strawberry', '*****@*****.**']])
modifier = SetModifier(self.reader,
fieldNames=['name'],
source=source,
fn=fn)
modifier.initialize()
records = [r for r in modifier.getRecords()]
modifier.finalize()
self.assertEqual(len(records), 85)
self.assertEqual(
len([
r for r in records
if r.getField('name').getValue() == 'Found same email address'
]), 6)
self.assertEqual(
len([
r for r in records if r.getField('name').getValue() ==
'Not found same email address'
]), 79)
def test_is_in_source_condition( self ):
false_source_records = [
FieldSet([
Field( 'integer', IntegerFieldType(), defaultValue = 1 ),
Field( 'float', FloatFieldType(), defaultValue = 4.2 ),
Field( 'date', DateFieldType(), defaultValue = '2012-02-04' )
]),
FieldSet([
Field( 'integer', IntegerFieldType(), defaultValue = 2 ),
Field( 'float', FloatFieldType(), defaultValue = 4.3 ),
Field( 'date', DateFieldType(), defaultValue = '2012-02-05' )
]),
FieldSet([
Field( 'integer', IntegerFieldType(), defaultValue = 3 ),
Field( 'float', FloatFieldType(), defaultValue = 4.4 ),
Field( 'date', DateFieldType(), defaultValue = '2012-02-06' )
])
]
self.assertFalse( IsInSourceCondition( false_source_records, 'integer' ).getResult( self.integer_field ) )
self.assertFalse( IsInSourceCondition( false_source_records, 'float' ).getResult( self.float_field ) )
self.assertFalse( IsInSourceCondition( false_source_records, 'date' ).getResult( self.date_field ) )
true_source_records = [
FieldSet([
Field( 'integer', IntegerFieldType(), defaultValue = 3 ),
Field( 'float', FloatFieldType(), defaultValue = 4.31 ),
Field( 'date', DateFieldType(), defaultValue = '2012-02-04' )
]),
FieldSet([
Field( 'integer', IntegerFieldType(), defaultValue = 4 ),
Field( 'float', FloatFieldType(), defaultValue = 4.32 ),
Field( 'date', DateFieldType(), defaultValue = '2013-02-05' )
]),
FieldSet([
Field( 'integer', IntegerFieldType(), defaultValue = 5 ),
Field( 'float', FloatFieldType(), defaultValue = 4.33 ),
Field( 'date', DateFieldType(), defaultValue = '2012-02-06' )
])
]
self.assertTrue( IsInSourceCondition( true_source_records, 'integer' ).getResult( self.integer_field ) )
self.assertTrue( IsInSourceCondition( true_source_records, 'float' ).getResult( self.float_field ) )
self.assertTrue( IsInSourceCondition( true_source_records, 'date' ).getResult( self.date_field ) )
class Test_Statement(unittest.TestCase):
def setUp(self):
self.field = Field('name', StringFieldType())
self.field.setTransforms([
IFStatement(IsEqualCondition('if_test')),
SetTransform('if_test_valid'),
ELIFStatement(IsEqualCondition('elif_test')),
SetTransform('elif_test_valid'), ELSEStatement,
SetTransform('else_test_valid'), ENDIFStatement,
ReturnTrueStatement
])
self.other_field = Field('name', StringFieldType())
self.other_field.setTransforms([
IFNotStatement(IsEqualCondition('if_not_test')),
IFNotStatement(IsEqualCondition('if_if_not_test')),
SetTransform('if_if_not_test_valid'),
ELIFNotStatement(IsEqualCondition('if_elif_not_test')),
SetTransform('if_elif_not_test_valid'), ENDIFStatement,
ENDIFStatement, ReturnTrueStatement
])
def test_if(self):
self.field.setValue('if_test')
self.field.run()
self.assertEqual(self.field.getValue(), 'if_test_valid')
def test_elif(self):
self.field.setValue('elif_test')
self.field.run()
self.assertEqual(self.field.getValue(), 'elif_test_valid')
def test_else(self):
self.field.setValue('else_test')
self.field.run()
self.assertEqual(self.field.getValue(), 'else_test_valid')
def test_if_not(self):
self.other_field.setValue('if_if_not_test')
self.other_field.run()
self.assertEqual(self.other_field.getValue(), 'if_elif_not_test_valid')
def test_elif_not(self):
self.other_field.setValue('if_elif_not_test')
self.other_field.run()
self.assertEqual(self.other_field.getValue(), 'if_if_not_test_valid')
