def fetch_report_from_gads_client_customer_obj(self, report_definition,
client_customer_id):
if not self.valid_client_customer_id(client_customer_id):
raise ClickException(
f"Invalid format: {client_customer_id}. Client customer ID should respect the following format 123-456-7890."
)
else:
try:
adwords_client = self.init_adwords_client(client_customer_id)
report_downloader = adwords_client.GetReportDownloader()
customer_report = report_downloader.DownloadReportAsStream(
report_definition,
client_customer_id=client_customer_id,
include_zero_impressions=self.include_zero_impressions,
skip_report_header=True,
skip_column_header=True,
skip_report_summary=True,
)
return customer_report
except AdWordsReportBadRequestError as e:
if e.type == "AuthorizationError.CUSTOMER_NOT_ACTIVE":
logger.warning(
f"Skipping clientCustomerId {client_customer_id} (inactive)."
)
else:
raise Exception(f"Wrong request. Error type: {e.type}")
def add_period_to_params(self, params):
"""
Add the time_increment, time_range and/or date_preset keys to parameters.
- time_increment: available in Ad Insights queries
- time_range and date_preset: available in Ad Insights queries,
and in Ad Management queries at the campaign, adset or ad levels only
"""
if self.ad_insights and self.time_increment:
params["time_increment"] = self.time_increment
if self.ad_insights or self.level in ["campaign", "adset", "ad"]:
if self.start_date and self.end_date:
logger.info(
"Date format used for request: start_date and end_date")
params["time_range"] = self.create_time_range()
elif self.date_preset:
logger.info("Date format used for request: date_preset")
params["date_preset"] = self.date_preset
else:
logging.warning(
"No date range provided - Last 30 days by default")
logging.warning(
"https://developers.facebook.com/docs/marketing-api/reference/ad-account/insights#parameters"
)
logger.warning(
"No date range provided - Last 30 days by default")
logger.warning(
"https://developers.facebook.com/docs/marketing-api/reference/ad-account/insights#parameters"
)
def get_date_range(start_date=None, end_date=None):
if start_date and end_date:
start = start_date.strftime("%Y-%m-%d")
end = end_date.strftime("%Y-%m-%d")
logger.warning(f"Custom date range selected: {start} --> {end}")
return {"startDate": start, "endDate": end}
else:
raise SyntaxError(
"Please provide start date and end date in your request")
def result_generator():
while True:
try:
pub = next(all_publications)
yield dict(pub)
except StopIteration:
break
except Exception:
ex_type, ex, tb = sys.exc_info()
logger.warning(
f"Failed to ingest post with error: {ex}. Traceback: {traceback.print_tb(tb)}"
)
def decode_if_needed(line):
if isinstance(line, bytes):
try:
line = line.decode("utf-8")
except UnicodeDecodeError as e:
logger.warning(
"An error has occurred while parsing the file."
f"The line could not be decoded in {e.encoding}."
f"Invalid input that the codec failed on: {e.object[e.start : e.end]}"
)
line = line.decode("utf-8", errors="ignore")
return line
def get_report_generator_from_flat_file(
line_iterator,
delimiter=",",
skip_n_first=0,
skip_n_last=0,
add_column=False,
column_dict={},
):
"""
From the line iterator of a flat file:
[
"Date,AdvertiserId,Impressions",
"2020-01-01,1234,10",
"2020-01-01,5678,20"
]
Return a generator of {column: value} dictionnaries:
[
{"Date": "2020-01-01", "AdvertiserId": "1234", "Impressions": "10"},
{"Date": "2020-01-01", "AdvertiserId": "5678", "Impressions": "20"}
]
Params
:line_iterator (iter): line iterator of the file to process
:delimiter (str): delimiter to parse file lines
:skip_n_first (int): nb of lines to skip at begining of file (excl. blank lines)
:skip_n_last (int): nb of lines to skip at end of file (excl. blank lines)
:add_column (bool): wether to add a fixed {column: value} at the end of each record
:column_dict (dict): if add_column is True, {column: value} dictionnary
to add at the end of each record (can include multiple column_names)
"""
first_line = True
for line in skip(line_iterator, skip_n_first, skip_n_last):
line = decode_if_needed(line)
if first_line:
first_line = False
headers = parse_decoded_line(line, delimiter)
else:
parsed_line = parse_decoded_line(line, delimiter)
if len(parsed_line) != len(headers):
logger.warning(
f"Skipping line '{line}': length of parsed line doesn't match length of headers."
)
else:
record = dict(zip(headers, parsed_line))
if add_column:
yield {**record, **column_dict}
else:
yield record
def get_date_range_for_ga_request(self):
start_date = self.kwargs.get("start_date")
end_date = self.kwargs.get("end_date")
date_range = self.kwargs.get("date_range")
day_range = self.kwargs.get("day_range")
if start_date and end_date:
logger.info("Date format used for request : startDate and endDate")
return self.create_date_range(start_date, end_date)
elif date_range:
logger.info("Date format used for request : dateRange")
return self.create_date_range(date_range[0], date_range[1])
elif day_range:
logger.info("Date format used for request : dayRange")
return self.generate_date_range_with_day_range(day_range)
else:
logger.warning("No date range provided - Last 7 days by default")
return []
def throttle(self):
"""
Monitoring API rate limit (12 requests every 6 seconds).
"""
current_time = time.time()
self.ingestion_tracker.append(current_time)
window_ingestion_tracker = [
t for t in self.ingestion_tracker
if t >= (current_time - API_WINDOW_DURATION)
]
if len(window_ingestion_tracker) >= API_REQUESTS_OVER_WINDOW_LIMIT:
sleep_time = window_ingestion_tracker[
0] + API_WINDOW_DURATION - current_time
logger.warning(
f"Throttling activated: sleeping for {sleep_time} seconds...")
time.sleep(sleep_time)
def log_sampling(report):
""" Log sampling data if a report has been sampled."""
data = report.get("data", {})
if data.get("samplesReadCounts") is not None:
logger.warning("☝️Report has been sampled.")
sample_reads = data["samplesReadCounts"][0]
sample_space = data["samplingSpaceSizes"][0]
logger.warning(f"sample reads : {sample_reads}")
logger.warning(f"sample space :{sample_space}")
logger.warning(
f"sample percent :{100 * int(sample_reads) / int(sample_space)}%"
)
else:
logger.info("Report is not sampled.")
def get_reach_report(self):
"""
Get 'REACH' report through the 'Reach and Average Frequency' endpoint of Twitter Ads API.
Documentation: https://developer.twitter.com/en/docs/ads/analytics/api-reference/reach
"""
resource = f"/{API_VERSION}/stats/accounts/{self.account.id}/reach/{self.entity.lower()}s"
entity_ids = self.get_active_entity_ids()
for chunk_entity_ids in split_list(entity_ids, MAX_ENTITY_IDS_PER_JOB):
try:
params = {
"account_id": self.account.id,
f"{self.entity.lower()}_ids": ",".join(entity_ids),
"start_time": self.start_date.strftime(API_DATEFORMAT),
"end_time": self.end_date.strftime(API_DATEFORMAT),
}
request = Request(self.client, "get", resource, params=params)
yield from Cursor(None, request)
except Exception:
ex_type, ex, tb = sys.exc_info()
logger.warning(
f"Failed to ingest post with error: {ex}. Traceback: {traceback.print_tb(tb)}"
)
def get_date_range(start_date, end_date):
start = start_date.strftime("%Y-%m-%d")
end = end_date.strftime("%Y-%m-%d")
logger.warning(f"Custom date range selected: {start} --> {end}")
return {"startDate": start, "endDate": end}
def check_end_date(end_date):
if end_date > MAX_END_DATE:
logger.warning(
f"The most recent date you can request is {datetime.strftime(MAX_END_DATE, DATEFORMAT)}"
)
return end_date