def test_get_report_generator__skip_when_no_match_with_headers_length(
self):
lines = [
b"Date,AdvertiserId,Impressions",
b"2020-01-01,1234,10",
b"2020-01-01,5678,20",
b"Copyrigth: report downloaded from Artefact.com",
]
line_iterator = iter(lines)
output = get_report_generator_from_flat_file(line_iterator,
skip_n_first=0,
skip_n_last=0)
expected = [
{
"Date": "2020-01-01",
"AdvertiserId": "1234",
"Impressions": "10"
},
{
"Date": "2020-01-01",
"AdvertiserId": "5678",
"Impressions": "20"
},
]
for output_record, expected_record in zip(output, expected):
self.assertEqual(output_record, expected_record)
def test_get_report_generator__add_column(self):
lines = [
b"Date,AdvertiserId,Reach",
b"2020-01-01,1234,1000",
b"2020-01-01,5678,2000",
]
expected = [
{
"Date": "2020-01-01",
"AdvertiserId": "1234",
"Reach": "1000",
"Campaign": "XMas Sale",
"Country": "France"
},
{
"Date": "2020-01-01",
"AdvertiserId": "5678",
"Reach": "2000",
"Campaign": "XMas Sale",
"Country": "France"
},
]
line_iterator = iter(lines)
output = get_report_generator_from_flat_file(
line_iterator,
add_column=True,
column_dict={
"Campaign": "XMas Sale",
"Country": "France"
},
)
for output_record, expected_record in zip(output, expected):
self.assertEqual(output_record, expected_record)
def test_get_report_generator__skip_last_with_blank_at_end_of_file(self):
lines = [
b"Date,AdvertiserId,Impressions",
b"2020-01-01,1234,10",
b"2020-01-01,5678,20",
b"(Not desired last line)",
b"",
]
line_iterator = iter(lines)
output = get_report_generator_from_flat_file(line_iterator,
skip_n_first=0,
skip_n_last=1)
expected = [
{
"Date": "2020-01-01",
"AdvertiserId": "1234",
"Impressions": "10"
},
{
"Date": "2020-01-01",
"AdvertiserId": "5678",
"Impressions": "20"
},
]
for output_record, expected_record in zip(output, expected):
self.assertEqual(output_record, expected_record)
def test_get_report_generator__file_with_headers_only(self):
lines = [b"Just,Headers,in,this,empty,report"]
line_iterator = iter(lines)
self.assertFalse(
next(get_report_generator_from_flat_file(line_iterator), False),
"Data is not empty",
)
def get_lineitems_objects(self):
body_lineitems = self.get_lineitems_body()
response = self._client.lineitems().downloadlineitems(
body=body_lineitems).execute()
lineitems = response["lineItems"]
lines = lineitems.split("\n")
return get_report_generator_from_flat_file(lines, skip_n_last=1)
def result_generator(self):
api_client = ApiClient(self.token, YANDEX_DIRECT_API_BASE_URL)
body = self._build_request_body()
headers = self._build_request_headers()
while True:
response = api_client.execute_request(url="reports",
body=body,
headers=headers,
stream=True)
if response.status_code == HTTPStatus.CREATED:
waiting_time = int(response.headers["retryIn"])
logger.info(
f"Report added to queue. Should be ready in {waiting_time} min."
)
time.sleep(waiting_time * 60)
elif response.status_code == HTTPStatus.ACCEPTED:
logger.info("Report in queue.")
elif response.status_code == HTTPStatus.OK:
logger.info("Report successfully retrieved.")
return get_report_generator_from_flat_file(
response.iter_lines(),
delimiter="\t",
skip_n_first=1,
)
return get_report_generator_from_flat_file(
response.iter_lines(),
delimiter="\t",
skip_n_first=1,
)
elif response.status_code == HTTPStatus.BAD_REQUEST:
logger.error("Invalid request.")
logger.error(response.json())
break
elif response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR:
logger.error("Internal server error.")
logger.error(response.json())
break
else:
logger.error(response.json())
break
return None
def get_query_report(self, existing_query=True):
url = self.get_query_report_url(existing_query)
report = requests.get(url, stream=True)
if self.kwargs[
"query_param_type"] == "TYPE_REACH_AND_FREQUENCY" and self.kwargs[
"add_date_to_report"]:
start, stop = get_date_start_and_date_stop_from_date_range(
self.kwargs["day_range"])
column_dict = {
"date_start": start.strftime(self.kwargs.get("date_format")),
"date_stop": stop.strftime(self.kwargs.get("date_format")),
}
report_gen = get_report_generator_from_flat_file(
report.iter_lines(), add_column=True, column_dict=column_dict)
return skip_last(report_gen, 1)
else:
report_gen = get_report_generator_from_flat_file(
report.iter_lines())
return skip_last(report_gen, 1)
def test_get_report_generator__no_bytes(self):
lines = [
"Date,Country,AdvertiserId,Impressions",
"2020-01-01,France,1234,10"
]
line_iterator = iter(lines)
expected = {
"Date": "2020-01-01",
"Country": "France",
"AdvertiserId": "1234",
"Impressions": "10",
}
for output in get_report_generator_from_flat_file(line_iterator):
self.assertDictEqual(output, expected)
def test_get_report_generator__multiple_encodings(self):
test_string_to_encode = "2020-01-01,France,1234,10"
lines = [
b"Date,Country,AdvertiserId,Impressions",
test_string_to_encode.encode("utf-8"),
test_string_to_encode.encode("ascii"),
test_string_to_encode.encode("windows-1252"),
test_string_to_encode.encode("latin_1"),
]
line_iterator = iter(lines)
expected = {
"Date": "2020-01-01",
"Country": "France",
"AdvertiserId": "1234",
"Impressions": "10",
}
for output in get_report_generator_from_flat_file(line_iterator):
self.assertDictEqual(output, expected)
def result_generator(self):
for advertiser_id in self.advertiser_ids:
body = self.sa360_client.generate_report_body(
self.agency_id,
advertiser_id,
self.report_type,
self.columns,
self.start_date,
self.end_date,
self.saved_columns,
)
report_id = self.sa360_client.request_report_id(body)
report_data = self.sa360_client.assert_report_file_ready(report_id)
for line_iterator in self.sa360_client.download_report_files(
report_data, report_id):
yield from get_report_generator_from_flat_file(line_iterator)
def test_get_report_generator__invalid_byte(self):
lines = [
b"Date,Country,AdvertiserId,Impressions",
b'2020-01-01," \x91\xea\xd0$",1234,10',
]
line_iterator = iter(lines)
expected = {
"Date": "2020-01-01",
"Country": " $",
"AdvertiserId": "1234",
"Impressions": "10",
}
with self.assertLogs(level=logging.INFO) as log:
for output in get_report_generator_from_flat_file(line_iterator):
self.assertDictEqual(output, expected)
self.assertEqual(
log.output,
[
"WARNING:root:An error has occurred while parsing the file."
"The line could not be decoded in utf-8."
"Invalid input that the codec failed on: b'\\x91'"
],
)
def _download_report(self):
report = requests.get(url=self.download_url,
headers=self.headers,
stream=True)
return get_report_generator_from_flat_file(report.iter_lines())