def test_listexpander( self ): static_source = StaticSource( FieldSet([ Field( 'namelist', ListFieldType() ), Field( 'email', StringFieldType() ), Field( 'year', IntegerFieldType() ), Field( 'name', StringFieldType() ) ], FieldMap({ 'namelist': 0, 'email': 1, 'year': 2 })) ) static_source.setResource([ [ list('El Agent'), 'El [email protected]', 2008 ], [ list('Serious Electron'), 'Serious [email protected]', 2008 ], [ list('Brave Wizard'), 'Brave [email protected]', 2008 ] ]) expand = ListExpanderExpand( static_source, 'namelist', 'name' ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), len(list('El Agent')) + len(list('Serious Electron')) + len(list('Brave Wizard')) ) self.assertEqual( records[-1].getField('name').getValue(), 'd' ) self.assertEqual( records[-1].getField('email').getValue(), 'Brave [email protected]' )
def setUp( self ): self.reader = StaticSource( FieldSet([ Field( 'name', StringFieldType() ), Field( 'email', StringFieldType() ) ], FieldMap({ 'name': 0, 'email': 1 })) ) self.reader.setResource([ [ 'Ochala Wild', 'Ochala [email protected]' ], [ 'Sina Venomous', 'Sina [email protected]' ], [ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ], [ 'Sermak Bad', 'Sermak [email protected]' ], [ 'Olivia Deadly Dawod', 'Olivia Deadly [email protected]' ], [ 'Pendus Inhuman', 'Pendus [email protected]' ], [ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ], [ 'Shard Brutal', 'Shard [email protected]' ], [ 'Sina Cruel', 'Sina [email protected]' ], [ 'Deadly Ohmar', 'Deadly [email protected]' ], [ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ], [ 'Calden Frigid', 'Calden [email protected]' ], [ 'Acid Reaper', 'Acid [email protected]' ], [ 'Raven Seth', 'Raven [email protected]' ], [ 'Rivatha Todal', 'Rivatha [email protected]' ], [ 'Panic Oliviaezit', 'Panic [email protected]' ], [ 'Tomara Wild', 'Tomara [email protected]' ], [ 'Venessa Metalhead', 'Venessa [email protected]' ] ])
def test_listexpander_map( self ): static_source = StaticSource( FieldSet([ Field( 'city', StringFieldType() ), Field( 'geometry', ListFieldType() ), Field( 'latitude', FloatFieldType() ), Field( 'longitude', FloatFieldType() ) ], FieldMap({ 'city': 0, 'geometry': 1 })) ) static_source.setResource([ [ 'Balatonlelle', [[17.6874552,46.7871465],[17.6865955,46.7870049],[17.6846158,46.7866786],[17.6834977,46.7864944],[17.6822251,46.7862847],[17.6815319,46.7861705],[17.6811473,46.7861071],[17.6795989,46.785852],[17.6774482,46.7854976],[17.6739061,46.7849139],[17.6729351,46.7847539],[17.6720789,46.7846318]] ], [ 'Balatonlelle', [[17.6871206,46.7880197],[17.6874552,46.7871465]] ], [ 'Balatonboglar', "[[17.6709959,46.7843474],[17.6710995,46.7840514],[17.6711443,46.7838791],[17.6711725,46.7837746],[17.6713146,46.7831483]]" ] ]) expand = ListExpanderExpand( static_source, 'geometry', expanderMap = { 'latitude': 1, 'longitude': 0 } ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), 19 ) self.assertEqual( records[10].getField('city').getValue(), 'Balatonlelle' ) self.assertEqual( records[10].getField('latitude').getValue(), 46.7847539 ) self.assertEqual( records[-1].getField('city').getValue(), 'Balatonboglar' ) self.assertEqual( records[-1].getField('longitude').getValue(), 17.6713146 )
def setUp( self ): self.field = Field( 'name', StringFieldType() ) self.field.setTransforms([ IFStatement( IsEqualCondition('if_test') ), SetTransform( 'if_test_valid' ), ELIFStatement( IsEqualCondition('elif_test') ), SetTransform( 'elif_test_valid' ), ELSEStatement, SetTransform( 'else_test_valid'), ENDIFStatement, ReturnTrueStatement ]) self.other_field = Field( 'name', StringFieldType() ) self.other_field.setTransforms([ IFNotStatement( IsEqualCondition('if_not_test') ), IFNotStatement( IsEqualCondition('if_if_not_test') ), SetTransform('if_if_not_test_valid'), ELIFNotStatement( IsEqualCondition('if_elif_not_test') ), SetTransform('if_elif_not_test_valid'), ENDIFStatement, ENDIFStatement, ReturnTrueStatement ])
def test_is_match_by_regexp_condition( self ): match_field = Field( 'string', StringFieldType(), defaultValue = 'It will match, because it has 1 or more number value.' ) not_match_field = Field( 'string', StringFieldType(), defaultValue = 'It will match, because it has one or more number value.' ) self.assertTrue( IsMatchByRegexpCondition( r'^.*(\d+).*$' ).getResult( match_field ) ) self.assertFalse( IsMatchByRegexpCondition( r'^.*(\d+).*$' ).getResult( not_match_field ) ) self.assertFalse( IsMatchByRegexpCondition( r'^.*(\d+).*$' ).getResult( self.empty_str_field ) )
def test_joinbykey_modifier(self): source = StaticSource( FieldSet([ Field('email', StringFieldType(), key=True), Field('age', IntegerFieldType()) ], FieldMap({ 'email': 0, 'age': 1 }))) source.setResource( [['El [email protected]', 12], ['Ochala [email protected]', 14], ['Sina [email protected]', 17], ['Akassa Savage [email protected]', 16], ['Sermak [email protected]', 22], ['Olivia Deadly [email protected]', 32], ['*****@*****.**', 42], ['Naria Cold-blodded [email protected]', 22], ['*****@*****.**', 54], ['Sina [email protected]', 56], ['Deadly [email protected]', 43], ['Mylenedriz [email protected]', 23], ['Calden [email protected]', 35], ['*****@*****.**', 56], ['Raven [email protected]', 23], ['*****@*****.**', 45], ['Pluto [email protected]', 64], ['Southern [email protected]', 53], ['Serious [email protected]', 62], ['*****@*****.**', 63], ['Risky [email protected]', 21], ['Rivatha [email protected]', 56], ['Panic [email protected]', 25], ['Tomara [email protected]', 46], ['Venessa [email protected]', 53], ['Western [email protected]', 71], ['*****@*****.**', 76]]) records = [ r for r in JoinByKeyModifier(self.reader, fieldNames=['age'], source=source).initialize().getRecords() ] self.assertEqual(len(records), 85) self.assertIsNone(records[-1].getField('age').getValue()) self.assertEqual(records[-2].getField('age').getValue(), 71) self.assertEqual(records[0].getField('age').getValue(), 12)
def test_remove_words_by_source(self): sourceRecords = [ FieldSet( [Field('name', StringFieldType(), defaultValue='killing')]), FieldSet( [Field('name', StringFieldType(), defaultValue='sadness')]), FieldSet( [Field('name', StringFieldType(), defaultValue='hungarian')]) ] self.field.setValue( u'contains hungarian members attractive sadness killing') self.field.setTransforms([RemoveWordsBySourceTransform(sourceRecords) ]).run() self.assertEqual(self.field.getValue(), 'contains members attractive')
def setUp(self): self.fs = FieldSet([ Field('id', IntegerFieldType(), defaultValue=1, key=True), Field('name', StringFieldType(), defaultValue='John doe'), Field('email', StringFieldType(), key=True), Field('birth_date', DateFieldType(), field_final_type=StringFieldType()), Field('favourite_number', FloatFieldType()), Field('created', DateTimeFieldType(), defaultValue='2013-04-04 18:44:32'), Field('updated', DateTimeFieldType(), defaultValue='2013-04-04 18:44:32') ]) self.fm = FieldMap({ 'id': 0, 'name': 1, 'email': 2, 'birth_date': 3, 'favourite_number': 4, 'created': 5, 'updated': 6, 'not_existing_field_name': 6 })
def setUp(self): self.field = Field('name', StringFieldType()) self.field.setTransforms([ IFStatement(IsEqualCondition('if_test')), SetTransform('if_test_valid'), ELIFStatement(IsEqualCondition('elif_test')), SetTransform('elif_test_valid'), ELSEStatement, SetTransform('else_test_valid'), ENDIFStatement, ReturnTrueStatement ]) self.other_field = Field('name', StringFieldType()) self.other_field.setTransforms([ IFNotStatement(IsEqualCondition('if_not_test')), IFNotStatement(IsEqualCondition('if_if_not_test')), SetTransform('if_if_not_test_valid'), ELIFNotStatement(IsEqualCondition('if_elif_not_test')), SetTransform('if_elif_not_test_valid'), ENDIFStatement, ENDIFStatement, ReturnTrueStatement ])
def test_field_manipulation(self): with self.assertRaises(FieldNotExistsError): self.fs.deleteField('key') with self.assertRaises(TypeError): self.fs.addField('key') self.assertEqual(len(self.fs.getFields()), 7) self.fs.deleteField('birth_date') self.assertEqual(len(self.fs.getFields()), 6) self.fs.addField(Field('language', StringFieldType())) self.assertEqual(len(self.fs.getFields()), 7)
def setUp( self ): self.integer_field = Field( 'integer', IntegerFieldType(), defaultValue = 4 ) self.float_field = Field( 'float', FloatFieldType(), defaultValue = 4.32 ) self.date_field = Field( 'date', DateFieldType(), defaultValue = '2013-02-05' ) self.string_field = Field( 'string', StringFieldType(), defaultValue = 'test' ) self.empty_int_field = Field( 'integer_empty', IntegerFieldType() ) self.empty_str_field = Field( 'string_empty', StringFieldType(), defaultValue = '' )
def test_append( self ): source = CSVSource( FieldSet( fields = [ Field( 'NUMBERS', StringFieldType() ) ], fieldmap = FieldMap({ 'NUMBERS': 0 }) ) ) source.setResource('tests/test_sources/test_csv_append.csv') expand = AppendExpand( source, resource = 'tests/test_sources/test_csv_appended.csv' ) expand.initialize() records = [ r for r in expand.getRecords() ] expand.finalize() self.assertEqual( len( records ), 6 ) self.assertEqual( records[0].getField('NUMBERS').getValue(), 'First' ) self.assertEqual( records[-1].getField('NUMBERS').getValue(), 'Sixth' )
def test_field( self ): static_source = StaticSource( FieldSet([ Field( 'year', IntegerFieldType() ), Field( 'country', StringFieldType() ), Field( 'count', IntegerFieldType() ), Field( 'cz', IntegerFieldType() ), Field( 'hu', IntegerFieldType() ), Field( 'sk', IntegerFieldType() ), Field( 'pl', IntegerFieldType() ) ], FieldMap({ 'year': 0, 'cz': 1, 'hu': 2, 'sk': 3, 'pl': 4 })) ) static_source.setResource([ [1999,32,694,129,230], [1999,395,392,297,453], [1999,635,812,115,97] ]) expand = FieldExpand( static_source, fieldNamesAndLabels = { 'cz': 'Czech', 'hu': 'Hungary', 'sk': 'Slovak', 'pl': 'Poland', }, valueFieldName = 'count', labelFieldName = 'country' ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), 12 ) self.assertEqual( records[-1].getField('year').getValue(), 1999 ) self.assertEqual( records[-1].getField('count').getValue(), 812 ) self.assertEqual( records[-1].getField('pl').getValue(), 97 ) self.assertEqual( records[-1].getField('country').getValue(), 'Hungary' )
def test_melt( self ): static_source = StaticSource( FieldSet([ Field( 'first', StringFieldType() ), Field( 'height', FloatFieldType() ), Field( 'last', StringFieldType() ), Field( 'weight', IntegerFieldType() ), Field( 'iq', IntegerFieldType() ), Field( 'quantity', StringFieldType() ), Field( 'value', FloatFieldType() ) ], FieldMap({ 'first': 0, 'height': 1, 'last': 2, 'weight': 3, 'iq': 4 })) ) static_source.setResource([ ['John',5.5,'Doe',130,102], ['Mary',6.0,'Bo',150,98] ]) expand = MeltExpand( static_source, fieldNames = ['first','last'], valueFieldName = 'value', labelFieldName = 'quantity' ).initialize() records = [ r for r in expand.getRecords() ] self.assertEqual( len( records ), 6 ) self.assertEqual( len( records[0].getFieldNames() ), 4) self.assertEqual( records[-1].getField('first').getValue(), 'Mary' ) self.assertEqual( records[-1].getField('last').getValue(), 'Bo' ) self.assertEqual( records[-1].getField('quantity').getValue(), 'height' ) self.assertEqual( records[-1].getField('value').getValue(), 6.0 )
def test_replace_words_by_source(self): sourceRecords = [ FieldSet([ Field('from', StringFieldType(), defaultValue='killing'), Field('to', StringFieldType(), defaultValue='born') ]), FieldSet([ Field('from', StringFieldType(), defaultValue='sadness'), Field('to', StringFieldType(), defaultValue='happyness') ]), FieldSet([ Field('from', StringFieldType(), defaultValue='hungarian'), Field('to', StringFieldType(), defaultValue='anywhere else') ]) ] self.field.setValue( u'contains hungarian members attractive sadness killing') self.field.setTransforms( [ReplaceWordsBySourceTransform(sourceRecords, join='from')]).run() self.assertEqual( self.field.getValue(), 'contains anywhere else members attractive happyness born')
def setUp(self): self.reader = StaticSource( FieldSet([ Field('name', StringFieldType()), Field('email', StringFieldType()), Field('year', IntegerFieldType()), Field('after_year', IntegerFieldType()), Field('number', IntegerFieldType()) ], FieldMap({ 'name': 0, 'email': 1, 'year': 2, 'after_year': 3 }))) self.reader.setResource( [['El Agent', 'El [email protected]', 2008, 2008], [ 'Serious Electron', 'Serious [email protected]', 2008, 2013 ], ['Brave Wizard', 'Brave [email protected]', 2008, 2008], [ 'Forgotten Itchy Emperor', 'Forgotten Itchy [email protected]', 2008, 2013 ], [ 'The Moving Monkey', 'The Moving [email protected]', 2008, 2008 ], [ 'Evil Ghostly Brigadier', 'Evil Ghostly [email protected]', 2008, 2013 ], [ 'Strangely Oyster', 'Strangely [email protected]', 2008, 2008 ], [ 'Anaconda Silver', 'Anaconda [email protected]', 2006, 2008 ], ['Hawk Tough', 'Hawk [email protected]', 2004, 2008], [ 'The Disappointed Craw', 'The Disappointed [email protected]', 2008, 2013 ], ['The Raven', 'The [email protected]', 1999, 2008], [ 'Ruby Boomerang', 'Ruby [email protected]', 2008, 2008 ], ['Skunk Tough', 'Skunk [email protected]', 2010, 2008], [ 'The Nervous Forgotten Major', 'The Nervous Forgotten [email protected]', 2008, 2013 ], [ 'Bursting Furious Puppet', 'Bursting Furious [email protected]', 2011, 2008 ], ['Neptune Eagle', 'Neptune [email protected]', 2011, 2013], ['The Skunk', 'The [email protected]', 2008, 2013], ['Lone Demon', 'Lone [email protected]', 2008, 2008], ['The Skunk', 'The [email protected]', 1999, 2008], [ 'Gamma Serious Spear', 'Gamma Serious [email protected]', 2008, 2008 ], [ 'Sleepy Dirty Sergeant', 'Sleepy Dirty [email protected]', 2008, 2008 ], ['Red Monkey', 'Red [email protected]', 2008, 2008], [ 'Striking Tiger', 'Striking [email protected]', 2005, 2008 ], ['Sliding Demon', 'Sliding [email protected]', 2011, 2008], [ 'Lone Commander', 'Lone [email protected]', 2008, 2013 ], ['Dragon Insane', 'Dragon [email protected]', 2013, 2013], ['Demon Skilled', 'Demon [email protected]', 2011, 2004], ['Vulture Lucky', 'Vulture [email protected]', 2003, 2008], ['The Ranger', 'The [email protected]', 2013, 2008], ['Morbid Snake', 'Morbid [email protected]', 2011, 2008], [ 'Dancing Skeleton', 'Dancing [email protected]', 2001, 2004 ], ['The Psycho', 'The [email protected]', 2005, 2008], ['Jupiter Rider', 'Jupiter [email protected]', 2011, 2008], ['Green Dog', 'Green [email protected]', 2011, 2008], [ 'Brutal Wild Colonel', 'Brutal Wild [email protected]', 2004, 2008 ], ['Random Leader', 'Random [email protected]', 2008, 2008], [ 'Pluto Brigadier', 'Pluto [email protected]', 2008, 2004 ], [ 'Southern Kangaroo', 'Southern [email protected]', 2008, 2008 ], ['Serious Flea', 'Serious [email protected]', 2001, 2005], [ 'Nocturnal Raven', 'Nocturnal [email protected]', 2008, 2004 ], ['Risky Flea', 'Risky [email protected]', 2005, 2005], ['The Corporal', 'The [email protected]', 2013, 2008], [ 'The Lucky Barbarian', 'The Lucky [email protected]', 2008, 2008 ], [ 'Rocky Serious Dog', 'Rocky Serious [email protected]', 2008, 2008 ], [ 'The Frozen Guardian', 'The Frozen [email protected]', 2008, 2008 ], [ 'Freaky Frostbite', 'Freaky [email protected]', 2008, 2004 ], [ 'The Tired Raven', 'The Tired [email protected]', 2008, 2008 ], [ 'Disappointed Frostbite', 'Disappointed [email protected]', 2008, 2008 ], ['The Craw', 'The [email protected]', 2003, 2008], [ 'Gutsy Strangely Chief', 'Gutsy Strangely [email protected]', 2008, 2008 ], ['Queen Angry', 'Queen [email protected]', 2008, 2008], [ 'Pluto Albatross', 'Pluto [email protected]', 2003, 2008 ], [ 'Endless Invader', 'Endless [email protected]', 2003, 2004 ], [ 'Beta Young Sergeant', 'Beta Young [email protected]', 2008, 2011 ], ['The Demon', 'The [email protected]', 2003, 2008], ['Lone Monkey', 'Lone [email protected]', 2011, 2008], [ 'Bursting Electron', 'Bursting [email protected]', 2003, 2010 ], [ 'Gangster Solid', 'Gangster [email protected]', 2005, 2009 ], ['The Gladiator', 'The [email protected]', 2001, 2002], [ 'Flash Frostbite', 'Flash [email protected]', 2005, 2004 ], [ 'The Rainbow Pluto Demon', 'The Rainbow Pluto [email protected]', 2011, 2013 ], [ 'Poseidon Rider', 'Poseidon [email protected]', 2008, 2006 ], [ 'The Old Alpha Brigadier', 'The Old Alpha [email protected]', 2008, 2008 ], [ 'Rough Anaconda', 'Rough [email protected]', 2001, 2011 ], [ 'Tough Dinosaur', 'Tough [email protected]', 2011, 2010 ], [ 'The Lost Dinosaur', 'The Lost [email protected]', 2008, 2008 ], ['The Raven', 'The [email protected]', 2005, 2009], ['The Agent', 'The [email protected]', 2011, 2008], [ 'Brave Scarecrow', 'Brave [email protected]', 2008, 2007 ], [ 'Flash Skeleton', 'Flash [email protected]', 2008, 2006 ], ['The Admiral', 'The [email protected]', 1998, 2005], ['The Tombstone', 'The [email protected]', 2013, 2008], ['Golden Arrow', 'Golden [email protected]', 2008, 2005], [ 'White Guardian', 'White [email protected]', 2011, 2004 ], [ 'The Black Eastern Power', 'The Black Eastern [email protected]', 2008, 2008 ], [ 'Ruthless Soldier', 'Ruthless [email protected]', 2008, 2008 ], ['Dirty Clown', 'Dirty [email protected]', 2008, 2008], ['Alpha Admiral', 'Alpha [email protected]', 2008, 2008], [ 'Lightning Major', 'Lightning [email protected]', 2008, 2008 ], [ 'The Rock Demon', 'The Rock [email protected]', 2008, 2001 ], ['Wild Tiger', 'Wild [email protected]', 2008, 2001], [ 'The Pointless Bandit', 'The Pointless [email protected]', 2008, 2008 ], ['The Sergeant', 'The [email protected]', 1998, 2002], ['Western Ogre', 'Western [email protected]', 1998, 2004], [ 'Sergeant Strawberry', 'Sergeant [email protected]', 2006, 2008 ]])
class Test_Statement( unittest.TestCase ): def setUp( self ): self.field = Field( 'name', StringFieldType() ) self.field.setTransforms([ IFStatement( IsEqualCondition('if_test') ), SetTransform( 'if_test_valid' ), ELIFStatement( IsEqualCondition('elif_test') ), SetTransform( 'elif_test_valid' ), ELSEStatement, SetTransform( 'else_test_valid'), ENDIFStatement, ReturnTrueStatement ]) self.other_field = Field( 'name', StringFieldType() ) self.other_field.setTransforms([ IFNotStatement( IsEqualCondition('if_not_test') ), IFNotStatement( IsEqualCondition('if_if_not_test') ), SetTransform('if_if_not_test_valid'), ELIFNotStatement( IsEqualCondition('if_elif_not_test') ), SetTransform('if_elif_not_test_valid'), ENDIFStatement, ENDIFStatement, ReturnTrueStatement ]) def test_if( self ): self.field.setValue('if_test') self.field.run() self.assertEqual( self.field.getValue(), 'if_test_valid' ) def test_elif( self ): self.field.setValue('elif_test') self.field.run() self.assertEqual( self.field.getValue(), 'elif_test_valid' ) def test_else( self ): self.field.setValue('else_test') self.field.run() self.assertEqual( self.field.getValue(), 'else_test_valid' ) def test_if_not( self ): self.other_field.setValue('if_if_not_test') self.other_field.run() self.assertEqual( self.other_field.getValue(), 'if_elif_not_test_valid' ) def test_elif_not( self ): self.other_field.setValue('if_elif_not_test') self.other_field.run() self.assertEqual( self.other_field.getValue(), 'if_if_not_test_valid' )
def setUp(self): self.field = Field('string', StringFieldType())
class Test_Transform(unittest.TestCase): def setUp(self): self.field = Field('string', StringFieldType()) def test_clean_transform(self): self.field.setValue( ' That is a good sentence, which is contains many english word! ') self.field.setTransforms([CleanTransform(replaces={'many': '1+'}) ]).run() self.assertEqual( self.field.getValue(), 'That is a good sentence which is contains 1+ english word') def test_converttype_transform_soft(self): self.field.setValue('2012-02-12') self.field.setFinalType(DateFieldType()) self.field.setTransforms([ConvertTypeTransform('Date')]).run() self.assertEqual(self.field.getValue(), datetime.date(2012, 2, 12)) def test_converttype_transform_soft_invalid(self): self.field.setValue('four miles') self.field.setFinalType(DateFieldType()) with self.assertRaises(FieldTypeError): self.field.setTransforms([ConvertTypeTransform('Date')]).run() def test_converttype_transform_hard(self): self.field.setValue('four miles') self.field.setFinalType(DateFieldType()) self.field.setTransforms([ConvertTypeTransform('Date', hard=True)]).run() self.assertIsNone(self.field.getValue()) def test_converttype_transform_hard_with_default(self): self.field.setValue('four miles') self.field.setFinalType(DateFieldType()) self.field.setTransforms([ ConvertTypeTransform('Date', hard=True, defaultValue=datetime.date.today()) ]).run() self.assertEqual(self.field.getValue(), datetime.date.today()) def test_homogenize_transform(self): self.field.setValue(u'árvíztűrőtükörfúrógépÁRVÍZTŰRŐTÜKÖRFÚRÓGÉP') self.field.setTransforms([HomogenizeTransform]).run() self.assertEqual(self.field.getValue(), 'arvizturotukorfurogeparvizturotukorfurogep') def test_lowercase_transform(self): self.field.setValue( u'That is a good sentence, which is contains many english word!') self.field.setTransforms([LowerCaseTransform]).run() self.assertEqual( self.field.getValue(), 'that is a good sentence, which is contains many english word!') def test_uppercase_transform(self): self.field.setValue( u'That is a good sentence, which is contains many english word!') self.field.setTransforms([UpperCaseTransform]).run() self.assertEqual( self.field.getValue(), 'THAT IS A GOOD SENTENCE, WHICH IS CONTAINS MANY ENGLISH WORD!') def test_strip_transform(self): self.field.setValue( u' That is a good sentence, which is contains many english word! ' ) self.field.setTransforms([StripTransform()]).run() self.assertEqual( self.field.getValue(), 'That is a good sentence, which is contains many english word!') def test_title_transform(self): self.field.setValue( u'That is a good sentence, which is contains many english word!') self.field.setTransforms([TitleTransform]).run() self.assertEqual( self.field.getValue(), 'That Is A Good Sentence, Which Is Contains Many English Word!') def test_stem_transform(self): self.field.setValue( u'contains hungarian members attractive sadness killing') self.field.setTransforms([StemTransform('English')]).run() self.assertEqual(self.field.getValue(), 'contain hungarian member attract sad kill') def test_set_transform(self): self.field.setTransforms( [SetTransform('That will be the final value')]).run() self.assertEqual(self.field.getValue(), 'That will be the final value') def test_set_transform_srintfs(self): self.field.setValue('Myself') self.field.setTransforms([SetTransform('%(self)s or the new string') ]).run() self.assertEqual(self.field.getValue(), 'Myself or the new string') def test_map_transform(self): self.field.setValue(u'Monkey') self.field.setTransforms([MapTransform({'monkey': 'animal'})]).run() self.assertEqual(self.field.getValue(), 'Monkey') def test_map_transform_ignore_case(self): self.field.setValue(u'Monkey') self.field.setTransforms( [MapTransform({'monkey': 'animal'}, ignorecase=True)]).run() self.assertEqual(self.field.getValue(), 'animal') def test_map_transform_not_used_else_value(self): self.field.setValue(u'monkey') self.field.setTransforms( [MapTransform({'monkey': 'animal'}, elseValue='other')]).run() self.assertEqual(self.field.getValue(), 'animal') def test_map_transform_used_else_value(self): self.field.setValue(u'Building') self.field.setTransforms( [MapTransform({'monkey': 'animal'}, elseValue='other')]).run() self.assertEqual(self.field.getValue(), 'other') def test_map_transform_used_else_clear(self): self.field.setValue(u'Building') self.field.setTransforms( [MapTransform({'monkey': 'animal'}, elseClear=True)]).run() self.assertIsNone(self.field.getValue()) def test_replace_by_regexp(self): self.field.setValue(u'2012-04-20') self.field.setTransforms([ ReplaceByRegexpTransform(regexp='^([0-9]{4}-[0-9]{2})-[0-9]{2}$', to='$1') ]).run() self.assertEqual(self.field.getValue(), '2012-04') def test_remove_words_by_source(self): sourceRecords = [ FieldSet( [Field('name', StringFieldType(), defaultValue='killing')]), FieldSet( [Field('name', StringFieldType(), defaultValue='sadness')]), FieldSet( [Field('name', StringFieldType(), defaultValue='hungarian')]) ] self.field.setValue( u'contains hungarian members attractive sadness killing') self.field.setTransforms([RemoveWordsBySourceTransform(sourceRecords) ]).run() self.assertEqual(self.field.getValue(), 'contains members attractive') def test_replace_words_by_source(self): sourceRecords = [ FieldSet([ Field('from', StringFieldType(), defaultValue='killing'), Field('to', StringFieldType(), defaultValue='born') ]), FieldSet([ Field('from', StringFieldType(), defaultValue='sadness'), Field('to', StringFieldType(), defaultValue='happyness') ]), FieldSet([ Field('from', StringFieldType(), defaultValue='hungarian'), Field('to', StringFieldType(), defaultValue='anywhere else') ]) ] self.field.setValue( u'contains hungarian members attractive sadness killing') self.field.setTransforms( [ReplaceWordsBySourceTransform(sourceRecords, join='from')]).run() self.assertEqual( self.field.getValue(), 'contains anywhere else members attractive happyness born') def test_split_transform_first(self): self.field.setValue( u'contains hungarian members attractive sadness killing') self.field.setTransforms([SplitTransform('0')]).run() self.assertEqual(self.field.getValue(), 'contains') def test_split_transform_last(self): self.field.setValue( u'contains hungarian members attractive sadness killing') self.field.setTransforms([SplitTransform('-1')]).run() self.assertEqual(self.field.getValue(), 'killing') def test_split_transform_interval(self): self.field.setValue( u'contains hungarian members attractive sadness killing') self.field.setTransforms([SplitTransform('1:-1')]).run() self.assertEqual(self.field.getValue(), 'hungarian members attractive sadness')
def setUp( self ): self.field = Field( 'string', StringFieldType() )
class Test_Transform( unittest.TestCase ): def setUp( self ): self.field = Field( 'string', StringFieldType() ) def test_clean_transform( self ): self.field.setValue(' That is a good sentence, which is contains many english word! ') self.field.setTransforms([ CleanTransform( replaces = { 'many': '1+' } ) ]).run() self.assertEqual( self.field.getValue(), 'That is a good sentence which is contains 1+ english word' ) def test_converttype_transform_soft( self ): self.field.setValue('2012-02-12') self.field.setFinalType( DateFieldType() ) self.field.setTransforms([ ConvertTypeTransform( 'Date' ) ]).run() self.assertEqual( self.field.getValue(), datetime.date( 2012, 2, 12 ) ) def test_converttype_transform_soft_invalid( self ): self.field.setValue('four miles') self.field.setFinalType( DateFieldType() ) with self.assertRaises( FieldTypeError ): self.field.setTransforms([ ConvertTypeTransform( 'Date' ) ]).run() def test_converttype_transform_hard( self ): self.field.setValue('four miles') self.field.setFinalType( DateFieldType() ) self.field.setTransforms([ ConvertTypeTransform( 'Date', hard = True ) ]).run() self.assertIsNone( self.field.getValue() ) def test_converttype_transform_hard_with_default( self ): self.field.setValue('four miles') self.field.setFinalType( DateFieldType() ) self.field.setTransforms([ ConvertTypeTransform( 'Date', hard = True, defaultValue = datetime.date.today() ) ]).run() self.assertEqual( self.field.getValue(), datetime.date.today() ) def test_homogenize_transform( self ): self.field.setValue(u'árvíztűrőtükörfúrógépÁRVÍZTŰRŐTÜKÖRFÚRÓGÉP') self.field.setTransforms([ HomogenizeTransform ]).run() self.assertEqual( self.field.getValue(), 'arvizturotukorfurogeparvizturotukorfurogep' ) def test_lowercase_transform( self ): self.field.setValue(u'That is a good sentence, which is contains many english word!') self.field.setTransforms([ LowerCaseTransform ]).run() self.assertEqual( self.field.getValue(), 'that is a good sentence, which is contains many english word!' ) def test_uppercase_transform( self ): self.field.setValue(u'That is a good sentence, which is contains many english word!') self.field.setTransforms([ UpperCaseTransform ]).run() self.assertEqual( self.field.getValue(), 'THAT IS A GOOD SENTENCE, WHICH IS CONTAINS MANY ENGLISH WORD!' ) def test_strip_transform( self ): self.field.setValue(u' That is a good sentence, which is contains many english word! ') self.field.setTransforms([ StripTransform() ]).run() self.assertEqual( self.field.getValue(), 'That is a good sentence, which is contains many english word!' ) def test_title_transform( self ): self.field.setValue(u'That is a good sentence, which is contains many english word!') self.field.setTransforms([ TitleTransform ]).run() self.assertEqual( self.field.getValue(), 'That Is A Good Sentence, Which Is Contains Many English Word!' ) def test_stem_transform( self ): self.field.setValue(u'contains hungarian members attractive sadness killing') self.field.setTransforms([ StemTransform( 'English' ) ]).run() self.assertEqual( self.field.getValue(), 'contain hungarian member attract sad kill' ) def test_set_transform( self ): self.field.setTransforms([ SetTransform( 'That will be the final value' ) ]).run() self.assertEqual( self.field.getValue(), 'That will be the final value' ) def test_set_transform_srintfs( self ): self.field.setValue('Myself') self.field.setTransforms([ SetTransform( '%(self)s or the new string' ) ]).run() self.assertEqual( self.field.getValue(), 'Myself or the new string' ) def test_map_transform( self ): self.field.setValue(u'Monkey') self.field.setTransforms([ MapTransform({ 'monkey': 'animal' }) ]).run() self.assertEqual( self.field.getValue(), 'Monkey' ) def test_map_transform_ignore_case( self ): self.field.setValue(u'Monkey') self.field.setTransforms([ MapTransform({ 'monkey': 'animal' }, ignorecase = True ) ]).run() self.assertEqual( self.field.getValue(), 'animal' ) def test_map_transform_not_used_else_value( self ): self.field.setValue(u'monkey') self.field.setTransforms([ MapTransform({ 'monkey': 'animal' }, elseValue = 'other' ) ]).run() self.assertEqual( self.field.getValue(), 'animal' ) def test_map_transform_used_else_value( self ): self.field.setValue(u'Building') self.field.setTransforms([ MapTransform({ 'monkey': 'animal' }, elseValue = 'other' ) ]).run() self.assertEqual( self.field.getValue(), 'other' ) def test_map_transform_used_else_clear( self ): self.field.setValue(u'Building') self.field.setTransforms([ MapTransform({ 'monkey': 'animal' }, elseClear = True ) ]).run() self.assertIsNone( self.field.getValue() ) def test_replace_by_regexp( self ): self.field.setValue(u'2012-04-20') self.field.setTransforms([ ReplaceByRegexpTransform( regexp = '^([0-9]{4}-[0-9]{2})-[0-9]{2}$', to = '$1' ) ]).run() self.assertEqual( self.field.getValue(), '2012-04' ) def test_remove_words_by_source( self ): sourceRecords = [ FieldSet([ Field( 'name', StringFieldType(), defaultValue = 'killing' ) ]), FieldSet([ Field( 'name', StringFieldType(), defaultValue = 'sadness' ) ]), FieldSet([ Field( 'name', StringFieldType(), defaultValue = 'hungarian' ) ]) ] self.field.setValue(u'contains hungarian members attractive sadness killing') self.field.setTransforms([ RemoveWordsBySourceTransform( sourceRecords ) ]).run() self.assertEqual( self.field.getValue(), 'contains members attractive' ) def test_replace_words_by_source( self ): sourceRecords = [ FieldSet([ Field( 'from', StringFieldType(), defaultValue = 'killing' ), Field( 'to', StringFieldType(), defaultValue = 'born' ) ]), FieldSet([ Field( 'from', StringFieldType(), defaultValue = 'sadness' ), Field( 'to', StringFieldType(), defaultValue = 'happyness' ) ]), FieldSet([ Field( 'from', StringFieldType(), defaultValue = 'hungarian' ), Field( 'to', StringFieldType(), defaultValue = 'anywhere else' ) ]) ] self.field.setValue(u'contains hungarian members attractive sadness killing') self.field.setTransforms([ ReplaceWordsBySourceTransform( sourceRecords, join = 'from' ) ]).run() self.assertEqual( self.field.getValue(), 'contains anywhere else members attractive happyness born' ) def test_split_transform_first( self ): self.field.setValue(u'contains hungarian members attractive sadness killing') self.field.setTransforms([ SplitTransform( '0' ) ]).run() self.assertEqual( self.field.getValue(), 'contains' ) def test_split_transform_last( self ): self.field.setValue(u'contains hungarian members attractive sadness killing') self.field.setTransforms([ SplitTransform( '-1' ) ]).run() self.assertEqual( self.field.getValue(), 'killing' ) def test_split_transform_interval( self ): self.field.setValue(u'contains hungarian members attractive sadness killing') self.field.setTransforms([ SplitTransform( '1:-1' ) ]).run() self.assertEqual( self.field.getValue(), 'hungarian members attractive sadness' )
def test_set_modifier_source(self): def setValue(record, field, scope): return 'Found same email address' \ if record.getField('email').getValue() in [ sr.getField('email').getValue() for sr in scope.getSourceRecords() ] \ else 'Not found same email address' fn = setValue source = StaticSource( FieldSet([ Field('name', StringFieldType()), Field('email', StringFieldType()) ], FieldMap({ 'name': 0, 'email': 1 }))) source.setResource( [['El Agent', 'El [email protected]'], ['Ochala Wild', 'Ochala [email protected]'], ['Sina Venomous', 'Sina [email protected]'], [ 'Akassa Savage Phalloz', 'Akassa Savage [email protected]' ], ['Sermak Bad', 'Sermak [email protected]'], ['Olivia Deadly Dawod', 'Olivia Deadly [email protected]'], ['Pendus Inhuman', '*****@*****.**'], [ 'Naria Cold-blodded Greste', 'Naria Cold-blodded [email protected]' ], ['Shard Brutal', '*****@*****.**'], ['Sina Cruel', 'Sina [email protected]'], ['Deadly Ohmar', 'Deadly [email protected]'], [ 'Mylenedriz Cold-blodded', 'Mylenedriz [email protected]' ], ['Calden Frigid', 'Calden [email protected]'], ['Acid Reaper', '*****@*****.**'], ['Raven Seth', 'Raven [email protected]'], ['Random Leader', '*****@*****.**'], ['Pluto Brigadier', 'Pluto [email protected]'], ['Southern Kangaroo', 'Southern [email protected]'], ['Serious Flea', 'Serious [email protected]'], ['Nocturnal Raven', '*****@*****.**'], ['Risky Flea', 'Risky [email protected]'], ['Rivatha Todal', 'Rivatha [email protected]'], ['Panic Oliviaezit', 'Panic [email protected]'], ['Tomara Wild', 'Tomara [email protected]'], ['Venessa Metalhead', 'Venessa [email protected]'], ['Western Ogre', 'Western [email protected]'], ['Sergeant Strawberry', '*****@*****.**']]) modifier = SetModifier(self.reader, fieldNames=['name'], source=source, fn=fn) modifier.initialize() records = [r for r in modifier.getRecords()] modifier.finalize() self.assertEqual(len(records), 85) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Found same email address' ]), 6) self.assertEqual( len([ r for r in records if r.getField('name').getValue() == 'Not found same email address' ]), 79)
def test_is_in_source_condition( self ): false_source_records = [ FieldSet([ Field( 'integer', IntegerFieldType(), defaultValue = 1 ), Field( 'float', FloatFieldType(), defaultValue = 4.2 ), Field( 'date', DateFieldType(), defaultValue = '2012-02-04' ) ]), FieldSet([ Field( 'integer', IntegerFieldType(), defaultValue = 2 ), Field( 'float', FloatFieldType(), defaultValue = 4.3 ), Field( 'date', DateFieldType(), defaultValue = '2012-02-05' ) ]), FieldSet([ Field( 'integer', IntegerFieldType(), defaultValue = 3 ), Field( 'float', FloatFieldType(), defaultValue = 4.4 ), Field( 'date', DateFieldType(), defaultValue = '2012-02-06' ) ]) ] self.assertFalse( IsInSourceCondition( false_source_records, 'integer' ).getResult( self.integer_field ) ) self.assertFalse( IsInSourceCondition( false_source_records, 'float' ).getResult( self.float_field ) ) self.assertFalse( IsInSourceCondition( false_source_records, 'date' ).getResult( self.date_field ) ) true_source_records = [ FieldSet([ Field( 'integer', IntegerFieldType(), defaultValue = 3 ), Field( 'float', FloatFieldType(), defaultValue = 4.31 ), Field( 'date', DateFieldType(), defaultValue = '2012-02-04' ) ]), FieldSet([ Field( 'integer', IntegerFieldType(), defaultValue = 4 ), Field( 'float', FloatFieldType(), defaultValue = 4.32 ), Field( 'date', DateFieldType(), defaultValue = '2013-02-05' ) ]), FieldSet([ Field( 'integer', IntegerFieldType(), defaultValue = 5 ), Field( 'float', FloatFieldType(), defaultValue = 4.33 ), Field( 'date', DateFieldType(), defaultValue = '2012-02-06' ) ]) ] self.assertTrue( IsInSourceCondition( true_source_records, 'integer' ).getResult( self.integer_field ) ) self.assertTrue( IsInSourceCondition( true_source_records, 'float' ).getResult( self.float_field ) ) self.assertTrue( IsInSourceCondition( true_source_records, 'date' ).getResult( self.date_field ) )
class Test_Statement(unittest.TestCase): def setUp(self): self.field = Field('name', StringFieldType()) self.field.setTransforms([ IFStatement(IsEqualCondition('if_test')), SetTransform('if_test_valid'), ELIFStatement(IsEqualCondition('elif_test')), SetTransform('elif_test_valid'), ELSEStatement, SetTransform('else_test_valid'), ENDIFStatement, ReturnTrueStatement ]) self.other_field = Field('name', StringFieldType()) self.other_field.setTransforms([ IFNotStatement(IsEqualCondition('if_not_test')), IFNotStatement(IsEqualCondition('if_if_not_test')), SetTransform('if_if_not_test_valid'), ELIFNotStatement(IsEqualCondition('if_elif_not_test')), SetTransform('if_elif_not_test_valid'), ENDIFStatement, ENDIFStatement, ReturnTrueStatement ]) def test_if(self): self.field.setValue('if_test') self.field.run() self.assertEqual(self.field.getValue(), 'if_test_valid') def test_elif(self): self.field.setValue('elif_test') self.field.run() self.assertEqual(self.field.getValue(), 'elif_test_valid') def test_else(self): self.field.setValue('else_test') self.field.run() self.assertEqual(self.field.getValue(), 'else_test_valid') def test_if_not(self): self.other_field.setValue('if_if_not_test') self.other_field.run() self.assertEqual(self.other_field.getValue(), 'if_elif_not_test_valid') def test_elif_not(self): self.other_field.setValue('if_elif_not_test') self.other_field.run() self.assertEqual(self.other_field.getValue(), 'if_if_not_test_valid')