def test_with(self): t = MockTarget("foo") with t.open('w') as b: b.write("bar") with t.open('r') as b: self.assertEqual(list(b), ['bar'])
def test_preprocessing(self, output_mock, input_mock): # -------- SET UP MOCK DATA ------------ output_target = MockTarget('corpus_out', format=luigi.format.Nop) input_target = MockTarget('corpus_in', format=luigi.format.Nop) output_mock.return_value = output_target input_mock.return_value = input_target with input_target.open('w') as fp: pickle.dump([ Doc("Ich bin der erste Post über ein Kulturinstitut" "in der Landeshauptstadt"), Doc("Ich bin der 2 Post über mit Bezug zur " "Landeshauptstadt. toll "), Doc("Trallala noch ein Post 2 zum Museum"), Doc("noch weitere Posts zum weitere testen." 'Barberini toll'), Doc("this document is in english") ], fp) # ------- RUN TASK UNDER TEST -------- task = TopicModelingPreprocessCorpus() task.run() # ------- INSPECT OUTPUT ------- with output_target.open("r") as fp: output = pickle.load(fp) # nosec self.assertEqual(len(output), 2) self.assertEqual(output[0].tokens, ['post', 'landeshauptstadt', 'toll']) self.assertEqual(output[1].tokens, ['weitere', 'weitere', 'toll'])
def test_bytes(self): t = MockTarget("foo", format=Nop) with t.open('wb') as b: b.write(b"bar") with t.open('rb') as b: self.assertEqual(list(b), [b'bar'])
def test_default_mode_value(self): t = MockTarget("foo") with t.open('w') as b: b.write("bar") with t.open() as b: self.assertEqual(list(b), ['bar'])
def test_bytes(self): t = MockTarget("foo", format=Nop) with t.open('wb') as b: b.write(b"bar") with t.open('rb') as b: self.assertEqual(list(b), [b'bar'])
def test_with(self): t = MockTarget("foo") with t.open('w') as b: b.write("bar") with t.open('r') as b: self.assertEqual(list(b), ['bar'])
def test_unicode(self): t = MockTarget("foo") with t.open('w') as b: b.write(u"bar") with t.open('r') as b: self.assertEqual(b.read(), u'bar')
def test_default_mode_value(self): t = MockTarget("foo") with t.open('w') as b: b.write("bar") with t.open() as b: self.assertEqual(list(b), ['bar'])
def test_unicode(self): t = MockTarget("foo") with t.open('w') as b: b.write(u"bar") with t.open('r') as b: self.assertEqual(b.read(), u'bar')
def test_post_performance_transformation(self, input_mock, output_mock, request_mock): self.db_connector.execute( '''INSERT INTO ig_post (ig_post_id) VALUES (0123456789), (9876543210)''', '''INSERT INTO ig_post_performance VALUES (0123456789, '2019-10-04', 5, 4, 3, 2, 1, 2, 1, 0, 1, 0)''') input_target = MockTarget('posts_in', format=UTF8) input_mock.return_value = input_target output_target = MockTarget('insights_out', format=UTF8) output_mock.return_value = output_target with input_target.open('w') as posts_target: with open(f'{IG_TEST_DATA}/post_expected.csv', 'r', encoding='utf-8') as posts_input: posts_target.write(posts_input.read()) with open(f'{IG_TEST_DATA}/post_insights_video_actual.json', 'r', encoding='utf-8') as json_video_in: input_video_insights = json_video_in.read() with open(f'{IG_TEST_DATA}/post_insights_no_video_actual.json', 'r', encoding='utf-8') as json_no_video_in: input_no_video_insights = json_no_video_in.read() with open(f'{IG_TEST_DATA}/post_insights_expected.csv', 'r', encoding='utf-8') as expected_data_in: expected_df = pd.read_csv(expected_data_in) def mock_video_json(): return json.loads(input_video_insights) def mock_no_video_json(): return json.loads(input_no_video_insights) mock_video_response = MagicMock(ok=True, json=mock_video_json) mock_no_video_response = MagicMock(ok=True, json=mock_no_video_json) request_mock.side_effect = [ mock_video_response, mock_no_video_response ] with freeze_time('2020-01-01 00:00:05'): self.task = instagram.FetchIgPostPerformance( columns=[ column[0] for column in instagram.IgPostPerformanceToDb().columns ], timespan=dt.timedelta(days=100000), table='ig_post_performance') self.task.run() with output_target.open('r') as output_data: output_df = pd.read_csv(output_data) pd.testing.assert_frame_equal(expected_df, output_df)
def test_1(self): t = MockTarget('test') p = t.open('w') print('test', file=p) p.close() q = t.open('r') self.assertEqual(list(q), ['test\n']) q.close()
def test_1(self): t = MockTarget('test') p = t.open('w') print('test', file=p) p.close() q = t.open('r') self.assertEqual(list(q), ['test\n']) q.close()
def test_scrape_order_contains(self, output_mock, input_mock): test_data = pd.read_csv( 'tests/test_data/gomus/scrape_order_contains_data.csv') html_file_names = [] all_order_ids = test_data['order_id'].drop_duplicates() for order_id in all_order_ids: new_html_task = FetchGomusHTML(url=f"/admin/orders/{order_id}") new_html_task.run() html_file_names.append(new_html_task.output().path) input_target = MockTarget('orders_htmls_out', format=UTF8) input_mock.return_value = input_target with input_target.open('w') as input_file: input_file.write('\n'.join(html_file_names)) output_target = MockTarget('scraped_order_contains_out', format=UTF8) output_mock.return_value = output_target # -- execute code under test -- ScrapeGomusOrderContains().run() # -- inspect results -- with output_target.open('r') as output_file: actual_output = pd.read_csv(output_file) self.assertEqual(len(test_data.index), len(actual_output.index)) for i in range(len(actual_output)): expected_row = test_data.iloc[i] actual_row = actual_output.iloc[i] # test if order stayed the same self.assertEqual(expected_row['order_id'], actual_row['order_id']) # test if scraped data is correct hash_string = ','.join([ str(actual_row['article_id']), str(actual_row['article_type']), str(actual_row['ticket']), str(actual_row['date']), str(actual_row['quantity']), str(actual_row['price']), str(actual_row['is_cancelled']) ]) actual_hash = mmh3.hash(hash_string, seed=self.hash_seed) self.assertEqual(actual_hash, expected_row['expected_hash'], msg=f"Scraper got wrong values:\n\ {str(actual_row) if sys.stdin.isatty() else 'REDACTED ON NON-TTY'}")
def test_exhibition_times(self, output_mock, requests_get_mock): output_target = MockTarget('exhibition_out', format=UTF8) output_mock.return_value = output_target with open('tests/test_data/gomus/exhibitions/exhibitions_actual.json', 'r', encoding='utf-8') as data_in: input_data = data_in.read() with open( 'tests/test_data/gomus/exhibitions/' 'exhibition_times_expected.csv', 'r', encoding='utf-8') as data_out: expected_data = data_out.read() # Overwrite requests 'get' return value to provide our test data def mock_json(): return json.loads(input_data) mock_response = MagicMock(ok=True, json=mock_json) requests_get_mock.return_value = mock_response FetchExhibitionTimes().run() with output_target.open('r') as output_data: self.assertEqual(expected_data, output_data.read())
def test_empty_tweet_performance(self, output_mock, raw_tweets_mock): output_target = MockTarget('perform_extracted_out', format=UTF8) output_mock.return_value = output_target with open( 'tests/test_data/twitter/empty_raw_tweets.csv', 'r', encoding='utf-8') as data_in: raw_tweets = data_in.read() with open( 'tests/test_data/twitter/empty_tweet_performance.csv', 'r', encoding='utf-8') as data_out: expected_performance = data_out.read() self.install_mock_target( raw_tweets_mock, lambda file: file.write(raw_tweets)) task = ExtractTweetPerformance(table='tweet_performance') task.run() with output_target.open('r') as output_file: output = output_file.read() self.assertEqual(output, expected_performance)
def test_extract_tweet_performance(self, output_mock, raw_tweets_mock): self.db_connector.execute(''' INSERT INTO tweet (tweet_id) VALUES ('1234567890123456789'), ('111111111111111111'), ('2222222222222222222') ''') output_target = MockTarget('perform_extracted_out', format=UTF8) output_mock.return_value = output_target with open('tests/test_data/twitter/raw_tweets.csv', 'r', encoding='utf-8') as data_in: raw_tweets = data_in.read() with open('tests/test_data/twitter/expected_tweet_performance.csv', 'r', encoding='utf-8') as data_out: expected_performance = data_out.read() self.install_mock_target(raw_tweets_mock, lambda file: file.write(raw_tweets)) task = ExtractTweetPerformance(table='tweet_performance') task.run() with output_target.open('r') as output_file: output = output_file.read() self.assertEqual( output.split('\n')[0], expected_performance.split('\n')[0]) for i in range(1, 3): self.assertEqual( # cutting away the timestamp output.split('\n')[i].split(';')[:-1], expected_performance.split('\n')[i].split(';')[:-1])
def test_post_transformation(self, fact_mock, output_mock, requests_get_mock): fact_target = MockTarget('facts_in', format=UTF8) fact_mock.return_value = fact_target output_target = MockTarget('post_out', format=UTF8) output_mock.return_value = output_target with open(f'{FB_TEST_DATA}/post_actual.json', 'r', encoding='utf-8') as data_in: input_data = data_in.read() with open(f'{FB_TEST_DATA}/post_expected.csv', 'r', encoding='utf-8') as data_out: expected_data = data_out.read() # Overwrite requests 'get' return value to provide our test data def mock_json(): return json.loads(input_data) mock_response = MagicMock(ok=True, json=mock_json) requests_get_mock.return_value = mock_response facebook.MuseumFacts().run() facebook.FetchFbPosts().run() with output_target.open('r') as output_data: self.assertEqual(expected_data, output_data.read())
def test_audience_gender_age_transformation(self, input_mock, output_mock, request_mock): fact_target = MockTarget('facts_in', format=UTF8) input_mock.return_value = fact_target output_target = MockTarget('post_out', format=UTF8) output_mock.return_value = output_target with open(f'{IG_TEST_DATA}/audience_gender_age_actual.json', 'r', encoding='utf-8') as data_in: input_data = data_in.read() with open(f'{IG_TEST_DATA}/audience_gender_age_expected.csv', 'r', encoding='utf-8') as data_out: expected_data = data_out.read() def mock_json(): return json.loads(input_data) mock_response = MagicMock(ok=True, json=mock_json) request_mock.return_value = mock_response instagram.MuseumFacts().run() with freeze_time('2020-01-01 00:00:05'): instagram.FetchIgAudienceGenderAge(columns=[ col[0] for col in instagram.IgAudienceGenderAgeToDb().columns ]).run() with output_target.open('r') as output_data: self.assertEqual(output_data.read(), expected_data)
def test_create_corpus(self, output_mock): # -------- SET UP MOCK DATA ------------ output_target = MockTarget('corpus_out', format=luigi.format.Nop) output_mock.return_value = output_target self.db_connector.execute( ''' INSERT INTO tweet(user_id,tweet_id,text,response_to,post_date) VALUES ('user_id', 'tweet_id', 'tweet text', NULL, '2020-05-24 10:56:21') ''', ''' INSERT INTO fb_post_comment(post_id,comment_id,post_date, text,is_from_museum,response_to) VALUES ('post1','comment1','2020-05-24 10:56:21', 'text1',false,NULL), ('post2','comment2','2018-05-24 10:56:21', 'text2',true,NULL) ''') # ------- RUN TASK UNDER TEST -------- task = TopicModelingCreateCorpus() task.run() # ------- INSPECT OUTPUT ------- with output_target.open("r") as fp: corpus = pickle.load(fp) # nosec self.assertEqual(len(corpus), 2) self.assertIsInstance(corpus[0], Doc) self.assertIsInstance(corpus[1], Doc)
def test_fetch_total_profile_metrics(self, fact_mock, output_mock, request_mock): fact_target = MockTarget('facts_in', format=UTF8) fact_mock.return_value = fact_target output_target = MockTarget('post_out', format=UTF8) output_mock.return_value = output_target with open(f'{IG_TEST_DATA}/total_profile_metrics_actual.json', 'r', encoding='utf-8') as data_in: input_data = data_in.read() with open(f'{IG_TEST_DATA}/total_profile_metrics_expected.csv', 'r', encoding='utf-8') as data_out: expected_data = data_out.read() def mock_json(): return json.loads(input_data) mock_response = MagicMock(ok=True, json=mock_json) request_mock.return_value = mock_response with freeze_time('2020-01-01 00:00:05'): self.run_task( instagram.FetchIgTotalProfileMetrics(columns=[ col[0] for col in instagram.IgTotalProfileMetricsToDb().columns ])) with output_target.open('r') as output_data: self.assertEqual(output_data.read(), expected_data)
def test_two(self, input_mock, output_mock): df_in0 = pd.DataFrame([[1, 'foo'], [2, 'bar']], columns=['a', 'b']) df_in1 = pd.DataFrame([[42, 'spam'], [1337, 'häm']], columns=['a', 'b']) input_mock.return_value = [ self.install_mock_target(MagicMock(), lambda file: df.to_csv(file, index=False)) for df in [df_in0, df_in1] ] output_target = MockTarget(str(self), format=UTF8) output_mock.return_value = output_target self.task = ConcatCsvs() self.run_task(self.task) with output_target.open('r') as output: df_out = pd.read_csv(output) df_expected = pd.DataFrame([ [1, 'foo'], [2, 'bar'], [42, 'spam'], [1337, 'häm'], ], columns=['a', 'b']) pd.testing.assert_frame_equal(df_expected, df_out)
def test_fetch_twitter(self, output_mock): output_target = MockTarget('raw_out', format=UTF8) output_mock.return_value = output_target # Dirty workaround for pandas's peculiarities regarding default values none = object() expected_tweet = { 'tweet_id': 1225435275301654531, 'text': "#MuseumBarberini is cool!", 'user_id': 1189538451097608193, 'parent_tweet_id': none, 'timestamp': '2020-02-06 16:05:11+01:00' } with freeze_time('2020-02-06'): # On this day our team's account has posted a related tweet # See https://twitter.com/bpfn2020/status/1225435275301654531 FetchTwitter(timespan=dt.timedelta(days=1)).run() with output_target.open('r') as output_file: output_df = pd.read_csv(output_file) output_df = output_df.fillna(none) filtered_df = output_df for key, value in expected_tweet.items(): filtered_df = filtered_df[filtered_df[key] == value] self.assertTrue( len(filtered_df) >= 1, f"Did not find any tweet with {key} = {value}")
def test_app_id(self, input_mock): input_target = MockTarget('museum_facts', format=UTF8) input_mock.return_value = input_target with input_target.open('w') as fp: json.dump({'ids': {'gplay': {'appId': 'some ID'}}}, fp) self.task._app_id = None app_id = FetchGplayReviews().app_id self.assertEqual(app_id, 'some ID')
def test_run(self, input_mock, output_mock, mock_fetch, mock_lang): input_target = MockTarget('museum_facts', format=UTF8) input_mock.return_value = input_target with input_target.open('w') as fp: json.dump( {'ids': {'gplay': { 'appId': 'com.barberini.museum.barberinidigital' }}}, fp ) output_target = MockTarget('gplay.gplay_reviews', format=UTF8) output_mock.return_value = output_target FetchGplayReviews().run() expected = pd.DataFrame([RESPONSE_ELEM_1_RENAMED_COLS]) with output_target.open('r') as output_file: actual = pd.read_csv(output_file) pd.testing.assert_frame_equal(expected, actual)
def test_one(self, input_mock, output_mock): df_in = pd.DataFrame([[1, 'foo'], [2, 'bar']], columns=['a', 'b']) self.install_mock_target(input_mock, lambda file: df_in.to_csv(file, index=False)) output_target = MockTarget(str(self)) output_mock.return_value = output_target self.task = ConcatCsvs() self.run_task(self.task) with output_target.open('r') as output: df_out = pd.read_csv(output) pd.testing.assert_frame_equal(df_in, df_out)
class GomusFormatTest(DatabaseTestCase): """The abstract base class for gomus format tests.""" def __init__(self, report, expected_format, *args, **kwargs): super().__init__(*args, **kwargs) self.report = report self.expected_format = expected_format def prepare_output_target(self, output_mock): self.output_target = MockTarget('data_out', format=UTF8) output_mock.return_value = [self.output_target] def fetch_gomus_report(self, suffix='_7days', sheet=[0]): self.run_task( FetchGomusReport(report=self.report, suffix=suffix, sheet_indices=sheet)) def check_format(self, skiprows=0, skipfooter=0): with self.output_target.open('r') as output_file: df = pd.read_csv(output_file, skipfooter=skipfooter, skiprows=skiprows, engine='python') for i in range(len(self.expected_format)): if df.columns[i] == 'Keine Daten vorhanden': break # this checks whether the columns are named right self.assertEqual(df.columns[i], self.expected_format[i][0]) df.apply(lambda x: self.check_type( x[self.expected_format[i][0]], self.expected_format[i][1]), axis=1) def check_type(self, data, expected_type): # To check if the data in the columns has the right type, # we try to converte the string into the expected type and # catch a ValueError or TypeError, if something goes wrong. # As we don't process data from type "STRING"/ just store # it as text, we don't have to explicitly check the type. try: if data == '': pass elif expected_type == 'FLOAT': float(data) elif expected_type == 'DATE': dt.datetime.strptime(data, '%d.%m.%Y') elif expected_type == 'TIME': dt.datetime.strptime(data, '%H:%M') except (ValueError, TypeError): self.assertTrue(False, f'{data} is not from type {expected_type}')
class DummyFileWrapper(luigi.Task): """Dummy task to write an output file.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.mock_target = MockTarget(f'DummyFileWrapperMock{hash(self)}', format=luigi.format.UTF8) csv = luigi.Parameter() def run(self): with self.mock_target.open('w') as input_file: input_file.write(self.csv) def output(self): return self.mock_target
def test_post_transformation(self, fact_mock, output_mock, request_mock): fact_target = MockTarget('facts_in', format=UTF8) fact_mock.return_value = fact_target output_target = MockTarget('post_out', format=UTF8) output_mock.return_value = output_target with open(f'{IG_TEST_DATA}/post_actual.json', 'r', encoding='utf-8') as data_in: input_data = data_in.read() expected_data = pd.read_csv(f'{IG_TEST_DATA}/post_expected.csv') request_mock.side_effect = lambda url: \ MagicMock(ok=True, json=lambda: json.loads(input_data)) self.run_task(instagram.FetchIgPosts()) with output_target.open('r') as output_data: actual_data = pd.read_csv(output_data) pd.testing.assert_frame_equal(actual_data, expected_data)
def iter_task(self, mock_codes, max_index): gen = self.task.run() dep = next(gen) for i, code in enumerate(mock_codes): self.assertIsInstance(dep, FetchGomusHTML) self.assertLessEqual(i, max_index) if 200 <= code < 300: target = MockTarget(f'quota_{i}.html') elif code in dep.ignored_status_codes: target = MockTarget(f'quota_{i}.html.error') else: raise ValueError("Unhandled status code") with target.open('w'): pass try: dep = gen.send(FailableTarget(target)) except StopIteration: dep = None break self.assertFalse(dep)
def test_audience_origin_transformation(self, input_mock, output_mock, request_mock): fact_target = MockTarget('facts_in', format=UTF8) input_mock.return_value = fact_target output_target = MockTarget('post_out', format=UTF8) output_mock.return_value = output_target with open(f'{IG_TEST_DATA}/audience_origin_actual.json', 'r', encoding='utf-8') as data_in: input_data = data_in.read() with open(f'{IG_TEST_DATA}/audience_origin_expected.csv', 'r', encoding='utf-8') as data_out: expected_data = data_out.read() def mock_json(): return json.loads(input_data) mock_response = MagicMock(ok=True, json=mock_json) request_mock.return_value = mock_response instagram.MuseumFacts().run() with freeze_time('2020-01-01 00:00:05'): # Use city mode for testing, though the # transformation is the same for countries # The only difference between the two is the received, # data, which cannot be tested here instagram.FetchIgAudienceOrigin(columns=[ col[0] for col in instagram.IgAudienceCityToDb().columns ], country_mode=False).run() with output_target.open('r') as output_data: self.assertEqual(output_data.read(), expected_data)
def prepare_post_performance_mocks(self, input_mock, output_mock, requests_get_mock, actual_json): input_target = MockTarget('posts_in', format=UTF8) input_mock.return_value = input_target output_target = MockTarget('insights_out', format=UTF8) output_mock.return_value = output_target with input_target.open('w') as posts_target: with open(f'{FB_TEST_DATA}/post_expected_single.csv', 'r', encoding='utf-8') as posts_input: posts_target.write(posts_input.read()) with open(f'{FB_TEST_DATA}/{actual_json}', 'r', encoding='utf-8') as json_in: input_json = json_in.read() def mock_json(): return json.loads(input_json) mock_response = MagicMock(ok=True, json=mock_json) requests_get_mock.return_value = mock_response return output_target
def test_post_transformation(self, fact_mock, output_mock, request_mock): fact_target = MockTarget('facts_in', format=UTF8) fact_mock.return_value = fact_target output_target = MockTarget('post_out', format=UTF8) output_mock.return_value = output_target with open(f'{IG_TEST_DATA}/post_actual.json', 'r', encoding='utf-8') as data_in: input_data = data_in.read() with open(f'{IG_TEST_DATA}/post_expected.csv', 'r', encoding='utf-8') as data_out: expected_data = data_out.read() def mock_json(): return json.loads(input_data) mock_response = MagicMock(ok=True, json=mock_json) request_mock.return_value = mock_response self.run_task(instagram.FetchIgPosts()) with output_target.open('r') as output_data: self.assertEqual(output_data.read(), expected_data)
def _touch(self, path): t = MockTarget(path) with t.open('w'): pass
def test_mode_none_error(self): t = MockTarget("foo") with self.assertRaises(TypeError): with t.open(None) as b: b.write("bar")
def _touch(self, path): t = MockTarget(path) with t.open('w'): pass
def test_mode_none_error(self): t = MockTarget("foo") with self.assertRaises(TypeError): with t.open(None) as b: b.write("bar")
def test_fetch_twitter(self, output_mock): """Integration test! We post a real tweet and then try to fetch it.""" output_target = MockTarget('raw_out', format=UTF8) output_mock.return_value = output_target sample = "TestBarberiniAnalyticsFetchTwitter" \ + self.generate_random_hex_string(12) text = ( f"This is an automated random tweet for integration testing of " "BarberiniAnalytics.\n\n" f"{sample}\n\n" "For more information, see: https://github.com/Museum-Barberini/" "Barberini-Analytics/blob/master/tests/test_twitter.py" ) # ARRANGE tweet = self.post_tweet(text) time.sleep(3) # Wait for the tweet to be processed # ACT FetchTwitter( query=sample, timespan=dt.timedelta(days=1) ).run() with output_target.open('r') as output_file: output_df = pd.read_csv(output_file) # Dirty workaround for pandas's peculiarities regarding default values none = object() output_df = output_df.fillna(none) output_df['text'] = output_df['text'].apply( lambda text: re.sub(r'\s+', ' ', text) ) output_df['timestamp'] = output_df['timestamp'].apply( date_parser.parse) # ASSERT expected_tweet = { 'tweet_id': tweet['id'], 'user_id': tweet['user_id'], 'parent_tweet_id': none, 'timestamp': tweet['created_at'] } expected_text = re.sub(r'\s+', ' ', tweet['text']) text_predicates = [ lambda text: text.startswith(expected_text.split('…')[0]), lambda text: sample in text ] filtered_df = output_df for key, value in expected_tweet.items(): previous_df = filtered_df filtered_df = filtered_df[filtered_df[key] == value] self.assertTrue( len(filtered_df) >= 1, f"Did not find any tweet with {key} = {value}, " f"values are: {previous_df[key]}") for index, predicate in enumerate(text_predicates): previous_df = filtered_df filtered_df = filtered_df[filtered_df['text'].apply(predicate)] self.assertTrue( len(filtered_df) >= 1, f"Did not find any tweet with text matching predicate {index}" f"\n\nValues are: {previous_df['text']}")