def test_dump_to_sqlite_db_dumps(self): if os.path.exists( os.path.dirname(os.path.abspath(__file__)) + '/data/output.db'): os.remove( os.path.dirname(os.path.abspath(__file__)) + '/data/output.db') field_list = [ 'id_str', 'coordinates.coordinates.0', 'coordinates.coordinates.1', 'user.id_str', 'user.lang', 'lang', 'text', 'user.screen_name', 'user.location', 'user.description', 'created_at', 'user.friends_count', 'user.followers_count', 'retweet_count', 'entities.urls.0.expanded_url', 'entities.urls.1.expanded_url', 'entities.urls.2.expanded_url', 'entities.urls.3.expanded_url', 'entities.urls.4.expanded_url', 'entities.hashtags.0.text', 'entities.hashtags.1.text' ] output_path = os.path.dirname( os.path.realpath(__file__)) + '/' + 'data/output.db' collection = BsonCollection( os.path.dirname(os.path.realpath(__file__)) + '/' + config['bson']['valid']) collection.dump_to_sqlite_db(output_path, field_list) self.assertTrue(os.path.getsize(output_path) > 0) if os.path.exists( os.path.dirname(os.path.abspath(__file__)) + '/data/output.db'): os.remove( os.path.dirname(os.path.abspath(__file__)) + '/data/output.db')
def test_dump_to_sqlite_db_dumps_the_right_stuff(self): if os.path.exists( os.path.dirname(os.path.abspath(__file__)) + '/data/output.db'): os.remove( os.path.dirname(os.path.abspath(__file__)) + '/data/output.db') field_list = [ 'id_str', 'coordinates.coordinates.0', 'coordinates.coordinates.1', 'user.id_str', 'user.lang', 'lang', 'text', 'user.screen_name', 'user.location', 'user.description', 'created_at', 'user.friends_count', 'user.followers_count', 'retweet_count', 'entities.urls.0.expanded_url', 'entities.urls.1.expanded_url', 'entities.urls.2.expanded_url', 'entities.urls.3.expanded_url', 'entities.urls.4.expanded_url', 'entities.hashtags.0.text', 'entities.hashtags.1.text' ] output_path = os.path.dirname( os.path.realpath(__file__)) + '/' + 'data/output.db' collection = BsonCollection( os.path.dirname(os.path.realpath(__file__)) + '/' + config['bson']['valid']) collection.dump_to_sqlite_db(output_path, field_list) con = sqlite3.connect(output_path) cur = con.cursor() row = [ elem for row in cur.execute("SELECT * FROM data LIMIT 1;") for elem in row ] con.close() self.assertTrue(len(row) > 0) self.assertEqual( set(row), set([ '661275583813431296', 'NULL', 'NULL', '379851447', 'en', 'de', 'Susan Lindauer, Rtd US Army LTC Potter: Jade Helm https://t.co/VA4bQRudLt #jadehelm #newworldorder #usa #tyranny #threat', 'Col_Connaughton', 'London UK', '#gaza #palestine #israel #BDS MAD EVIL ISRAEL MURDERS BABIES CIVILIANS to STEAL PALESTINIAN LAND RESOURCES with USA UK HELP. To stop my tweets, BLOCK or MUTE me', 'Mon Nov 02 20:15:59 +0000 2015', 2019, 3159, 0, 'https://www.youtube.com/watch?v=0nJqymxVpwc', 'NULL', 'NULL', 'NULL', 'NULL', 'jadehelm', 'newworldorder' ])) if os.path.exists( os.path.dirname(os.path.abspath(__file__)) + '/data/output.db'): os.remove( os.path.dirname(os.path.abspath(__file__)) + '/data/output.db')