def setUp(self): self.mongoDb = MongoDB(dbName="unittest", collection="tinyForum") # Fill the little MongoDB with test JSON lines self.resetMongoTestDb() self.mysqldb = MySQLDB(user='******', db='unittest') # Start with an empty result MySQL table for each test: self.mysqldb.dropTable('contents') # Fill the fake UserGrade table with records of course participants: self.resetMySQLUserListDb() # Instantiate a Forum scrubber without the # name of a bson file that contains forum # records. That 'None' for the bson file will # make the class understand that it's being # instantiated for a unit test. self.forumScrubberAnonymized = EdxForumScrubber(None, mysqlDbObj=self.mysqldb, forumTableName='contents', allUsersTableName='unittest.UserGrade') self.forumScrubberRelatable = EdxForumScrubber(None, mysqlDbObj=self.mysqldb, forumTableName='contents', allUsersTableName='unittest.UserGrade', allowAnonScreenName=True) self.forumScrubberClear = EdxForumScrubber(None, mysqlDbObj=self.mysqldb, forumTableName='contents', allUsersTableName='unittest.UserGrade', anonymize=False)
def setUp(self): self.mongoDb = MongoDB(dbName="unittest", collection="tinyForum") # Fill the little MongoDB with test JSON lines self.resetMongoTestDb() self.mysqldb = MySQLDB(mySQLUser='******', db='unittest') # Start with an empty result MySQL table for each test: self.mysqldb.dropTable('contents') # Fill the fake UserGrade table with records of course participants: self.resetMySQLUserListDb() # Instantiate a Forum scrubber without the # name of a bson file that contains forum # records. That 'None' for the bson file will # make the class understand that it's being # instantiated for a unit test. self.forumScrubberAnonymized = EdxForumScrubber(None, mysqlDbObj=self.mysqldb, forumTableName='contents', allUsersTableName='unittest.UserGrade') self.forumScrubberRelatable = EdxForumScrubber(None, mysqlDbObj=self.mysqldb, forumTableName='contents', allUsersTableName='unittest.UserGrade', allowAnonScreenName=True) self.forumScrubberClear = EdxForumScrubber(None, mysqlDbObj=self.mysqldb, forumTableName='contents', allUsersTableName='unittest.UserGrade', anonymize=False)
class TestForumEtl(unittest.TestCase): # Forum rows have the following columns: # type, anonymous, anonymous_to_peers, at_position_list, user_int_id, body, course_display_name, created_at, votes, count, down_count, up_count, up, down, comment_thread_id, parent_id, parent_ids, sk # Correct result for relationization of tinyForum.json # (in <projDir>/src/forum_etl/data). This result is anonymized and not relatable, # i.e. poster name UIDs use integers, while other tables use hashes: tinyForumGoldAnonymized = \ [ # poster Otto van Homberg: body is clean to start with: ('anon_screen_name_redacted','CommentThread', 'False', 'False', '[]', 5L, 'Harmless body', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 20), "{u'count': 10, u'point': -6, u'down_count': 8, u'up': [u'2', u'10'], u'down': [u'1', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], u'up_count': 2}", 10L, 8L, 2L, "['2', '10']", "['1', '3', '4', '5', '6', '7', '8', '9']", None, None, None, None), # poster Andreas Fritz: body has someone's email: ('anon_screen_name_redacted','Comment', 'False', 'False', '[]', 7L, ' Body with <emailRedac> email.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 10, u'point': -4, u'down_count': 7, u'up': [u'6', u'8', u'10'], u'down': [u'1', u'2', u'3', u'4', u'5', u'7', u'9'], u'up_count': 3}", 10L, 7L, 3L, "['6', '8', '10']", "['1', '2', '3', '4', '5', '7', '9']", '519461545924670200000001', None, '[]', '519461555924670200000006'), # poster Otto van Homberg: body has 'Otto': ('anon_screen_name_redacted','Comment', 'False', 'False', '[]', 5L, 'Body with poster name <nameRedac_anon_screen_name_redacted> embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007'), # poster Andreas Fritz: body has a phone number: ('anon_screen_name_redacted','Comment', 'False', 'False', '[]', 10L, 'Body with <phoneRedac> a phone number.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461545924670200000005', "[u'519461545924670200000005']", '519461545924670200000005-519461555924670200000008'), # poster Otto van Homberg: body has his screen name (otto_king): ('anon_screen_name_redacted','Comment', 'False', 'False', '[]', 5L, 'Body with poster screen name <nameRedac_anon_screen_name_redacted> embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007'), # poster Otto van Homberg: body has his full name (Otto van Homberg): ('anon_screen_name_redacted','Comment', 'False', 'False', '[]', 5L, 'Body with poster screen name <nameRedac_anon_screen_name_redacted> <nameRedac_anon_screen_name_redacted> <nameRedac_anon_screen_name_redacted> embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007') ] # Gold result for anonymization that allows relating to other tables (i.e. hashes are constant) tinyForumGoldRelatable = \ [ # poster Otto van Homberg: body is clean to start with: ('abc','CommentThread', 'False', 'False', '[]', 5L, 'Harmless body', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 20), "{u'count': 10, u'point': -6, u'down_count': 8, u'up': [u'2', u'10'], u'down': [u'1', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], u'up_count': 2}", 10L, 8L, 2L, "['2', '10']", "['1', '3', '4', '5', '6', '7', '8', '9']", None, None, None, None), # poster Andreas Fritz: body has someone's email: ('def','Comment', 'False', 'False', '[]', 7L, ' Body with <emailRedac> email.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 10, u'point': -4, u'down_count': 7, u'up': [u'6', u'8', u'10'], u'down': [u'1', u'2', u'3', u'4', u'5', u'7', u'9'], u'up_count': 3}", 10L, 7L, 3L, "['6', '8', '10']", "['1', '2', '3', '4', '5', '7', '9']", '519461545924670200000001', None, '[]', '519461555924670200000006'), # poster Otto van Homberg: body has 'Otto': ('abc','Comment', 'False', 'False', '[]', 5L, 'Body with poster name <nameRedac_abc> embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007'), # poster Andreas Fritz: body has a phone number: ('ghi','Comment', 'False', 'False', '[]', 10L, 'Body with <phoneRedac> a phone number.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461545924670200000005', "[u'519461545924670200000005']", '519461545924670200000005-519461555924670200000008'), # poster Otto van Homberg: body has his screen name (otto_king): ('abc','Comment', 'False', 'False', '[]', 5L, 'Body with poster screen name <nameRedac_abc> embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007'), # poster Otto van Homberg: body has his full name (Otto van Homberg): ('abc','Comment', 'False', 'False', '[]', 5L, 'Body with poster screen name <nameRedac_abc> <nameRedac_abc> <nameRedac_abc> embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007') ] # Gold result for non-anonymized forum: tinyForumGoldClear = \ [ # poster Otto van Homberg: body is clean to start with: ('otto_king','CommentThread', 'False', 'False', '[]', 5L, 'Harmless body', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 20), "{u'count': 10, u'point': -6, u'down_count': 8, u'up': [u'2', u'10'], u'down': [u'1', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], u'up_count': 2}", 10L, 8L, 2L, "['2', '10']", "['1', '3', '4', '5', '6', '7', '8', '9']", None, None, None, None), # poster Andreas Fritz: body has someone's email: ('fritzL','Comment', 'False', 'False', '[]', 7L, ' Body with [email protected] email.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 10, u'point': -4, u'down_count': 7, u'up': [u'6', u'8', u'10'], u'down': [u'1', u'2', u'3', u'4', u'5', u'7', u'9'], u'up_count': 3}", 10L, 7L, 3L, "['6', '8', '10']", "['1', '2', '3', '4', '5', '7', '9']", '519461545924670200000001', None, '[]', '519461555924670200000006'), # poster Otto van Homberg: body has 'Otto': ('otto_king','Comment', 'False', 'False', '[]', 5L, 'Body with poster name Otto embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007'), # poster Andreas Fritz: body has a phone number: ('bebeW','Comment', 'False', 'False', '[]', 10L, 'Body with 650-333-4567 a phone number.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461545924670200000005', "[u'519461545924670200000005']", '519461545924670200000005-519461555924670200000008'), # poster Otto van Homberg: body has his screen name (otto_king): ('otto_king','Comment', 'False', 'False', '[]', 5L, 'Body with poster screen name otto_king embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007'), # poster Otto van Homberg: body has his full name (Otto van Homberg): ('otto_king','Comment', 'False', 'False', '[]', 5L, 'Body with poster screen name Otto van Homberg embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007') ] def setUp(self): self.mongoDb = MongoDB(dbName="unittest", collection="tinyForum") # Fill the little MongoDB with test JSON lines self.resetMongoTestDb() self.mysqldb = MySQLDB(user='******', db='unittest') # Start with an empty result MySQL table for each test: self.mysqldb.dropTable('contents') # Fill the fake UserGrade table with records of course participants: self.resetMySQLUserListDb() # Instantiate a Forum scrubber without the # name of a bson file that contains forum # records. That 'None' for the bson file will # make the class understand that it's being # instantiated for a unit test. self.forumScrubberAnonymized = EdxForumScrubber(None, mysqlDbObj=self.mysqldb, forumTableName='contents', allUsersTableName='unittest.UserGrade') self.forumScrubberRelatable = EdxForumScrubber(None, mysqlDbObj=self.mysqldb, forumTableName='contents', allUsersTableName='unittest.UserGrade', allowAnonScreenName=True) self.forumScrubberClear = EdxForumScrubber(None, mysqlDbObj=self.mysqldb, forumTableName='contents', allUsersTableName='unittest.UserGrade', anonymize=False) def tearDown(self): self.mysqldb.close() @unittest.skipIf(not RUN_ALL_TESTS, 'Uncomment this decoration if RUN_ALL_TESTS is False, and you want to run just this test.') def testAnonymized(self): self.forumScrubberAnonymized.populateUserCache() self.forumScrubberAnonymized.forumMongoToRelational(self.mongoDb, self.mysqldb, 'contents') for rowNum, forumPost in enumerate(self.mysqldb.query('SELECT * FROM unittest.contents')): # print(str(rowNum) + ':' + str(forumPost)) self.assertEqual(TestForumEtl.tinyForumGoldAnonymized[rowNum], forumPost) @unittest.skipIf(not RUN_ALL_TESTS, 'Uncomment this decoration if RUN_ALL_TESTS is False, and you want to run just this test.') def testNonAnonymizedRelatable(self): self.forumScrubberRelatable.populateUserCache() self.forumScrubberRelatable.forumMongoToRelational(self.mongoDb, self.mysqldb, 'contents') for rowNum, forumPost in enumerate(self.mysqldb.query('SELECT * FROM unittest.contents')): # print(str(rowNum) + ':' + str(forumPost)) self.assertEqual(TestForumEtl.tinyForumGoldRelatable[rowNum], forumPost) @unittest.skipIf(not RUN_ALL_TESTS, 'Uncomment this decoration if RUN_ALL_TESTS is False, and you want to run just this test.') def testNonAnonymized(self): self.forumScrubberClear.populateUserCache() self.forumScrubberClear.forumMongoToRelational(self.mongoDb, self.mysqldb, 'contents') for rowNum, forumPost in enumerate(self.mysqldb.query('SELECT * FROM unittest.contents')): # print(str(rowNum) + ':' + str(forumPost)) self.assertEqual(TestForumEtl.tinyForumGoldClear[rowNum], forumPost) def resetMongoTestDb(self): self.mongoDb.clearCollection() # Use small, known forum collection: currDir = os.path.dirname(__file__) with open(os.path.join(currDir, 'data/tinyForum.json'), 'r') as jsonFd: for line in jsonFd: forumPost = json.loads(line) self.mongoDb.insert(forumPost) def resetMySQLUserListDb(self): ''' Prepare a MySQL table that mimicks EdxPrivate.UserGrade. ''' userGradeColSpecs = OrderedDict( { 'name' : 'varchar(255)', 'screen_name' : 'varchar(255)', 'grade' : 'int', 'course_id' : 'varchar(255)', 'distinction' : 'tinyint', 'status' : 'varchar(50)', 'user_int_id' : 'int(11)', 'anon_screen_name' : 'varchar(40)' }) self.mysqldb.dropTable('UserGrade') self.mysqldb.createTable('UserGrade', userGradeColSpecs) self.mysqldb.bulkInsert('UserGrade', ('name','screen_name','grade','course_id','distinction','status','user_int_id','anon_screen_name'), [ ('Otto van Homberg','otto_king',5,'oldCourse',0,'notpassing',5,'abc'), ('Andreas Fritz','fritzL',2,'newCourse',0,'notpassing',7,'def'), ('Bebe Winter', 'bebeW',10,'History of Baking',1,'passing',10,'ghi') ])
class TestForumEtl(unittest.TestCase): # Forum rows have the following columns: # type, anonymous, anonymous_to_peers, at_position_list, user_int_id, body, course_display_name, created_at, votes, count, down_count, up_count, up, down, comment_thread_id, parent_id, parent_ids, sk # Correct result for relationization of tinyForum.json # (in <projDir>/src/forum_etl/data). This result is anonymized and not relatable, # i.e. poster name UIDs use integers, while other tables use hashes: tinyForumGoldAnonymized = \ [ # poster Otto van Homberg: body is clean to start with: ('anon_screen_name_redacted','CommentThread', 'False', 'False', '[]', 5L, 'Harmless body', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 20), "{u'count': 10, u'point': -6, u'down_count': 8, u'up': [u'2', u'10'], u'down': [u'1', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], u'up_count': 2}", 10L, 8L, 2L, "['2', '10']", "['1', '3', '4', '5', '6', '7', '8', '9']", None, None, None, None), # poster Andreas Fritz: body has someone's email: ('anon_screen_name_redacted','Comment', 'False', 'False', '[]', 7L, ' Body with <emailRedac> email.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 10, u'point': -4, u'down_count': 7, u'up': [u'6', u'8', u'10'], u'down': [u'1', u'2', u'3', u'4', u'5', u'7', u'9'], u'up_count': 3}", 10L, 7L, 3L, "['6', '8', '10']", "['1', '2', '3', '4', '5', '7', '9']", '519461545924670200000001', None, '[]', '519461555924670200000006'), # poster Otto van Homberg: body has 'Otto': ('anon_screen_name_redacted','Comment', 'False', 'False', '[]', 5L, 'Body with poster name <nameRedac_anon_screen_name_redacted> embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007'), # poster Andreas Fritz: body has a phone number: ('anon_screen_name_redacted','Comment', 'False', 'False', '[]', 10L, 'Body with <phoneRedac> a phone number.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461545924670200000005', "[u'519461545924670200000005']", '519461545924670200000005-519461555924670200000008'), # poster Otto van Homberg: body has his screen name (otto_king): ('anon_screen_name_redacted','Comment', 'False', 'False', '[]', 5L, 'Body with poster screen name <nameRedac_anon_screen_name_redacted> embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007'), # poster Otto van Homberg: body has his full name (Otto van Homberg): ('anon_screen_name_redacted','Comment', 'False', 'False', '[]', 5L, 'Body with poster screen name <nameRedac_anon_screen_name_redacted> <nameRedac_anon_screen_name_redacted> <nameRedac_anon_screen_name_redacted> embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007') ] # Gold result for anonymization that allows relating to other tables (i.e. hashes are constant) tinyForumGoldRelatable = \ [ # poster Otto van Homberg: body is clean to start with: ('abc','CommentThread', 'False', 'False', '[]', 5L, 'Harmless body', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 20), "{u'count': 10, u'point': -6, u'down_count': 8, u'up': [u'2', u'10'], u'down': [u'1', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], u'up_count': 2}", 10L, 8L, 2L, "['2', '10']", "['1', '3', '4', '5', '6', '7', '8', '9']", None, None, None, None), # poster Andreas Fritz: body has someone's email: ('def','Comment', 'False', 'False', '[]', 7L, ' Body with <emailRedac> email.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 10, u'point': -4, u'down_count': 7, u'up': [u'6', u'8', u'10'], u'down': [u'1', u'2', u'3', u'4', u'5', u'7', u'9'], u'up_count': 3}", 10L, 7L, 3L, "['6', '8', '10']", "['1', '2', '3', '4', '5', '7', '9']", '519461545924670200000001', None, '[]', '519461555924670200000006'), # poster Otto van Homberg: body has 'Otto': ('abc','Comment', 'False', 'False', '[]', 5L, 'Body with poster name <nameRedac_abc> embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007'), # poster Andreas Fritz: body has a phone number: ('ghi','Comment', 'False', 'False', '[]', 10L, 'Body with <phoneRedac> a phone number.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461545924670200000005', "[u'519461545924670200000005']", '519461545924670200000005-519461555924670200000008'), # poster Otto van Homberg: body has his screen name (otto_king): ('abc','Comment', 'False', 'False', '[]', 5L, 'Body with poster screen name <nameRedac_abc> embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007'), # poster Otto van Homberg: body has his full name (Otto van Homberg): ('abc','Comment', 'False', 'False', '[]', 5L, 'Body with poster screen name <nameRedac_abc> <nameRedac_abc> <nameRedac_abc> embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007') ] # Gold result for non-anonymized forum: tinyForumGoldClear = \ [ # poster Otto van Homberg: body is clean to start with: ('otto_king','CommentThread', 'False', 'False', '[]', 5L, 'Harmless body', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 20), "{u'count': 10, u'point': -6, u'down_count': 8, u'up': [u'2', u'10'], u'down': [u'1', u'3', u'4', u'5', u'6', u'7', u'8', u'9'], u'up_count': 2}", 10L, 8L, 2L, "['2', '10']", "['1', '3', '4', '5', '6', '7', '8', '9']", None, None, None, None), # poster Andreas Fritz: body has someone's email: ('fritzL','Comment', 'False', 'False', '[]', 7L, ' Body with [email protected] email.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 10, u'point': -4, u'down_count': 7, u'up': [u'6', u'8', u'10'], u'down': [u'1', u'2', u'3', u'4', u'5', u'7', u'9'], u'up_count': 3}", 10L, 7L, 3L, "['6', '8', '10']", "['1', '2', '3', '4', '5', '7', '9']", '519461545924670200000001', None, '[]', '519461555924670200000006'), # poster Otto van Homberg: body has 'Otto': ('otto_king','Comment', 'False', 'False', '[]', 5L, 'Body with poster name Otto embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007'), # poster Andreas Fritz: body has a phone number: ('bebeW','Comment', 'False', 'False', '[]', 10L, 'Body with 650-333-4567 a phone number.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461545924670200000005', "[u'519461545924670200000005']", '519461545924670200000005-519461555924670200000008'), # poster Otto van Homberg: body has his screen name (otto_king): ('otto_king','Comment', 'False', 'False', '[]', 5L, 'Body with poster screen name otto_king embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007'), # poster Otto van Homberg: body has his full name (Otto van Homberg): ('otto_king','Comment', 'False', 'False', '[]', 5L, 'Body with poster screen name Otto van Homberg embedded.', 'MITx/6.002x/2012_Fall', datetime.datetime(2013, 5, 16, 4, 32, 21), "{u'count': 0, u'point': 0, u'down_count': 0, u'up': [], u'down': [], u'up_count': 0}", 0L, 0L, 0L, '[]', '[]', '519461545924670200000001', '519461555924670200000006', "[u'519461555924670200000006']", '519461555924670200000006-519461555924670200000007') ] def setUp(self): self.mongoDb = MongoDB(dbName="unittest", collection="tinyForum") # Fill the little MongoDB with test JSON lines self.resetMongoTestDb() self.mysqldb = MySQLDB(mySQLUser='******', db='unittest') # Start with an empty result MySQL table for each test: self.mysqldb.dropTable('contents') # Fill the fake UserGrade table with records of course participants: self.resetMySQLUserListDb() # Instantiate a Forum scrubber without the # name of a bson file that contains forum # records. That 'None' for the bson file will # make the class understand that it's being # instantiated for a unit test. self.forumScrubberAnonymized = EdxForumScrubber(None, mysqlDbObj=self.mysqldb, forumTableName='contents', allUsersTableName='unittest.UserGrade') self.forumScrubberRelatable = EdxForumScrubber(None, mysqlDbObj=self.mysqldb, forumTableName='contents', allUsersTableName='unittest.UserGrade', allowAnonScreenName=True) self.forumScrubberClear = EdxForumScrubber(None, mysqlDbObj=self.mysqldb, forumTableName='contents', allUsersTableName='unittest.UserGrade', anonymize=False) def tearDown(self): self.mysqldb.close() @unittest.skipIf(not RUN_ALL_TESTS, 'Uncomment this decoration if RUN_ALL_TESTS is False, and you want to run just this test.') def testAnonymized(self): self.forumScrubberAnonymized.populateUserCache() self.forumScrubberAnonymized.forumMongoToRelational(self.mongoDb, self.mysqldb, 'contents') for rowNum, forumPost in enumerate(self.mysqldb.query('SELECT * FROM unittest.contents')): # print(str(rowNum) + ':' + str(forumPost)) self.assertEqual(TestForumEtl.tinyForumGoldAnonymized[rowNum], forumPost) @unittest.skipIf(not RUN_ALL_TESTS, 'Uncomment this decoration if RUN_ALL_TESTS is False, and you want to run just this test.') def testNonAnonymizedRelatable(self): self.forumScrubberRelatable.populateUserCache() self.forumScrubberRelatable.forumMongoToRelational(self.mongoDb, self.mysqldb, 'contents') for rowNum, forumPost in enumerate(self.mysqldb.query('SELECT * FROM unittest.contents')): # print(str(rowNum) + ':' + str(forumPost)) self.assertEqual(TestForumEtl.tinyForumGoldRelatable[rowNum], forumPost) @unittest.skipIf(not RUN_ALL_TESTS, 'Uncomment this decoration if RUN_ALL_TESTS is False, and you want to run just this test.') def testNonAnonymized(self): self.forumScrubberClear.populateUserCache() self.forumScrubberClear.forumMongoToRelational(self.mongoDb, self.mysqldb, 'contents') for rowNum, forumPost in enumerate(self.mysqldb.query('SELECT * FROM unittest.contents')): # print(str(rowNum) + ':' + str(forumPost)) self.assertEqual(TestForumEtl.tinyForumGoldClear[rowNum], forumPost) def resetMongoTestDb(self): self.mongoDb.clearCollection() # Use small, known forum collection: currDir = os.path.dirname(__file__) with open(os.path.join(currDir, 'data/tinyForum.json'), 'r') as jsonFd: for line in jsonFd: forumPost = json.loads(line) self.mongoDb.insert(forumPost) def resetMySQLUserListDb(self): ''' Prepare a MySQL table that mimicks EdxPrivate.UserGrade. ''' userGradeColSpecs = OrderedDict( { 'name' : 'varchar(255)', 'screen_name' : 'varchar(255)', 'grade' : 'int', 'course_id' : 'varchar(255)', 'distinction' : 'tinyint', 'status' : 'varchar(50)', 'user_int_id' : 'int(11)', 'anon_screen_name' : 'varchar(40)' }) self.mysqldb.dropTable('UserGrade') self.mysqldb.createTable('UserGrade', userGradeColSpecs) self.mysqldb.bulkInsert('UserGrade', ('name','screen_name','grade','course_id','distinction','status','user_int_id','anon_screen_name'), [ ('Otto van Homberg','otto_king',5,'oldCourse',0,'notpassing',5,'abc'), ('Andreas Fritz','fritzL',2,'newCourse',0,'notpassing',7,'def'), ('Bebe Winter', 'bebeW',10,'History of Baking',1,'passing',10,'ghi') ])