def test_get_report_generator__skip_when_no_match_with_headers_length( self): lines = [ b"Date,AdvertiserId,Impressions", b"2020-01-01,1234,10", b"2020-01-01,5678,20", b"Copyrigth: report downloaded from Artefact.com", ] line_iterator = iter(lines) output = get_report_generator_from_flat_file(line_iterator, skip_n_first=0, skip_n_last=0) expected = [ { "Date": "2020-01-01", "AdvertiserId": "1234", "Impressions": "10" }, { "Date": "2020-01-01", "AdvertiserId": "5678", "Impressions": "20" }, ] for output_record, expected_record in zip(output, expected): self.assertEqual(output_record, expected_record)
def test_get_report_generator__add_column(self): lines = [ b"Date,AdvertiserId,Reach", b"2020-01-01,1234,1000", b"2020-01-01,5678,2000", ] expected = [ { "Date": "2020-01-01", "AdvertiserId": "1234", "Reach": "1000", "Campaign": "XMas Sale", "Country": "France" }, { "Date": "2020-01-01", "AdvertiserId": "5678", "Reach": "2000", "Campaign": "XMas Sale", "Country": "France" }, ] line_iterator = iter(lines) output = get_report_generator_from_flat_file( line_iterator, add_column=True, column_dict={ "Campaign": "XMas Sale", "Country": "France" }, ) for output_record, expected_record in zip(output, expected): self.assertEqual(output_record, expected_record)
def test_get_report_generator__skip_last_with_blank_at_end_of_file(self): lines = [ b"Date,AdvertiserId,Impressions", b"2020-01-01,1234,10", b"2020-01-01,5678,20", b"(Not desired last line)", b"", ] line_iterator = iter(lines) output = get_report_generator_from_flat_file(line_iterator, skip_n_first=0, skip_n_last=1) expected = [ { "Date": "2020-01-01", "AdvertiserId": "1234", "Impressions": "10" }, { "Date": "2020-01-01", "AdvertiserId": "5678", "Impressions": "20" }, ] for output_record, expected_record in zip(output, expected): self.assertEqual(output_record, expected_record)
def test_get_report_generator__file_with_headers_only(self): lines = [b"Just,Headers,in,this,empty,report"] line_iterator = iter(lines) self.assertFalse( next(get_report_generator_from_flat_file(line_iterator), False), "Data is not empty", )
def get_lineitems_objects(self): body_lineitems = self.get_lineitems_body() response = self._client.lineitems().downloadlineitems( body=body_lineitems).execute() lineitems = response["lineItems"] lines = lineitems.split("\n") return get_report_generator_from_flat_file(lines, skip_n_last=1)
def result_generator(self): api_client = ApiClient(self.token, YANDEX_DIRECT_API_BASE_URL) body = self._build_request_body() headers = self._build_request_headers() while True: response = api_client.execute_request(url="reports", body=body, headers=headers, stream=True) if response.status_code == HTTPStatus.CREATED: waiting_time = int(response.headers["retryIn"]) logger.info( f"Report added to queue. Should be ready in {waiting_time} min." ) time.sleep(waiting_time * 60) elif response.status_code == HTTPStatus.ACCEPTED: logger.info("Report in queue.") elif response.status_code == HTTPStatus.OK: logger.info("Report successfully retrieved.") return get_report_generator_from_flat_file( response.iter_lines(), delimiter="\t", skip_n_first=1, ) return get_report_generator_from_flat_file( response.iter_lines(), delimiter="\t", skip_n_first=1, ) elif response.status_code == HTTPStatus.BAD_REQUEST: logger.error("Invalid request.") logger.error(response.json()) break elif response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR: logger.error("Internal server error.") logger.error(response.json()) break else: logger.error(response.json()) break return None
def get_query_report(self, existing_query=True): url = self.get_query_report_url(existing_query) report = requests.get(url, stream=True) if self.kwargs[ "query_param_type"] == "TYPE_REACH_AND_FREQUENCY" and self.kwargs[ "add_date_to_report"]: start, stop = get_date_start_and_date_stop_from_date_range( self.kwargs["day_range"]) column_dict = { "date_start": start.strftime(self.kwargs.get("date_format")), "date_stop": stop.strftime(self.kwargs.get("date_format")), } report_gen = get_report_generator_from_flat_file( report.iter_lines(), add_column=True, column_dict=column_dict) return skip_last(report_gen, 1) else: report_gen = get_report_generator_from_flat_file( report.iter_lines()) return skip_last(report_gen, 1)
def test_get_report_generator__no_bytes(self): lines = [ "Date,Country,AdvertiserId,Impressions", "2020-01-01,France,1234,10" ] line_iterator = iter(lines) expected = { "Date": "2020-01-01", "Country": "France", "AdvertiserId": "1234", "Impressions": "10", } for output in get_report_generator_from_flat_file(line_iterator): self.assertDictEqual(output, expected)
def test_get_report_generator__multiple_encodings(self): test_string_to_encode = "2020-01-01,France,1234,10" lines = [ b"Date,Country,AdvertiserId,Impressions", test_string_to_encode.encode("utf-8"), test_string_to_encode.encode("ascii"), test_string_to_encode.encode("windows-1252"), test_string_to_encode.encode("latin_1"), ] line_iterator = iter(lines) expected = { "Date": "2020-01-01", "Country": "France", "AdvertiserId": "1234", "Impressions": "10", } for output in get_report_generator_from_flat_file(line_iterator): self.assertDictEqual(output, expected)
def result_generator(self): for advertiser_id in self.advertiser_ids: body = self.sa360_client.generate_report_body( self.agency_id, advertiser_id, self.report_type, self.columns, self.start_date, self.end_date, self.saved_columns, ) report_id = self.sa360_client.request_report_id(body) report_data = self.sa360_client.assert_report_file_ready(report_id) for line_iterator in self.sa360_client.download_report_files( report_data, report_id): yield from get_report_generator_from_flat_file(line_iterator)
def test_get_report_generator__invalid_byte(self): lines = [ b"Date,Country,AdvertiserId,Impressions", b'2020-01-01," \x91\xea\xd0$",1234,10', ] line_iterator = iter(lines) expected = { "Date": "2020-01-01", "Country": " $", "AdvertiserId": "1234", "Impressions": "10", } with self.assertLogs(level=logging.INFO) as log: for output in get_report_generator_from_flat_file(line_iterator): self.assertDictEqual(output, expected) self.assertEqual( log.output, [ "WARNING:root:An error has occurred while parsing the file." "The line could not be decoded in utf-8." "Invalid input that the codec failed on: b'\\x91'" ], )
def _download_report(self): report = requests.get(url=self.download_url, headers=self.headers, stream=True) return get_report_generator_from_flat_file(report.iter_lines())