def _process_single_day_jobs_concurrently(job_list: typing.List[JobQueueItem],
n_threads: int) -> None:
if n_threads < 1:
raise ValueError(
'_process_single_day_jobs_concurrently should have n_threads > 0')
heapq.heapify(job_list)
thread_args: ThreadArgs = ThreadArgs(job_list)
# store the default API since the worker threads will change it
default_api: FacebookAdsApi = FacebookAdsApi.get_default_api()
thread_list: typing.List[threading.Thread] = list()
thread: threading.Thread = threading.Thread(target=_retry_thread_func,
args=(thread_args, ))
thread_list.append(thread)
thread.start()
for i in range(0, n_threads):
thread = threading.Thread(target=_job_thread_func,
args=(thread_args, ))
thread_list.append(thread)
thread.start()
thread_args.state_changed_cv.acquire()
try:
while (not thread_args.error_occured) and (thread_args.jobs_left > 0):
thread_args.state_changed_cv.wait()
except:
thread_args.error_occured = True
finally:
thread_args.state_changed_cv.release()
# notify all waiting threads, so they can see that they are done
# release -> aquire ordering matters due to potential deadlocking
# use a second variable to identify a done variable
# that requires only a single lock rather than all three
with thread_args.job_list_cv:
thread_args.job_thread_done = True
thread_args.job_list_cv.notify_all()
with thread_args.retry_queue_cv:
thread_args.retry_thread_done = True
thread_args.retry_queue_cv.notify()
with thread_args.logging_mutex:
logging.info('waiting for all threads to exit'.format(
threading.get_ident()))
for thread in thread_list:
thread.join()
# restore the default API in case something else needs it after this function
FacebookAdsApi.set_default_api(default_api)
if thread_args.error_occured:
sys.exit(1)
def _job_thread_func(args: ThreadArgs) -> None:
job: typing.Optional[JobQueueItem]
# Api objects do not seem thread safe at all, create on per thread and nuke the default for
# good measure
api: FacebookAdsApi = FacebookAdsApi.init(config.app_id(),
config.app_secret(),
config.access_token())
FacebookAdsApi.set_default_api(None)
# No need to lock since this can only change to True, so worst case this does one extra
# iteration. Also assignment / reading of booleans should be atomic anyway.
while not args.done:
job = _get_job_from_queue(args)
if job is None:
continue
_process_job(args, job, api)
_log(logging.info, args.logging_mutex,
['thread {0} exited'.format(threading.get_ident())])
def _job_thread_func(args: ThreadArgs) -> None:
job: typing.Optional[JobQueueItem]
# Api objects do not seem thread safe at all, create on per thread and nuke the default for
# good measure
api: FacebookAdsApi = FacebookAdsApi.init(config.app_id(),
config.app_secret(),
config.access_token())
FacebookAdsApi.set_default_api(None)
with args.job_list_cv:
while not args.job_thread_done:
if len(args.job_list) > 0:
job = heapq.heappop(args.job_list)
args.job_list_cv.release()
_process_job(args, job, api)
args.job_list_cv.acquire()
else:
args.job_list_cv.wait()
_log(logging.info, args.logging_mutex,
['thread {0} exited'.format(threading.get_ident())])
def init(cls,
app_id=None,
app_secret=None,
access_token=None,
account_id=None,
api_version=None,
proxies=None,
timeout=None,
pool_maxsize=10,
max_retries=0):
# connection pool size is +1 because there also is the main thread
# that can also issue a request
session = FacebookSession(app_id, app_secret, access_token,
proxies=proxies, timeout=timeout,
pool_maxsize=pool_maxsize+1, max_retries=max_retries)
api = cls(session, api_version=api_version, threadpool_size=pool_maxsize)
cls.set_default_api(api)
# TODO: how to avoid this hack?
FacebookAdsApi.set_default_api(api)
if account_id:
cls.set_default_account_id(account_id)
def _eval_context_facebook(self, secrets, eval_context):
"""Adds Facebook SDK classes:
* FB.Application
* FB.Lead
Docs:
* https://github.com/facebook/facebook-python-business-sdk/tree/master/facebook_business/adobjects
The classes use App Access Token. To switch to other access levels, pass api= parameter on class instance initialization.
* FB_TOKEN.page
* FB_TOKEN.user
Token generation tools.
* FB_EXCHANGE_TOKEN.user2page
Docs:
* https://developers.facebook.com/docs/facebook-login/access-tokens
* https://developers.facebook.com/docs/pages/access-tokens
General methods for graph API:
* FB_GRAPH_API.app(method, url, **kwargs)
* FB_GRAPH_API.page(method, url, **kwargs)
* FB_GRAPH_API.user(method, url, **kwargs)
Docs:
* https://developers.facebook.com/docs/graph-api/
* https://docs.python-requests.org/en/latest/api/#requests.request
"""
log_transmission = eval_context["log_transmission"]
log = eval_context["log"]
LOG_INFO = eval_context["LOG_INFO"]
LOG_ERROR = eval_context["LOG_ERROR"]
LOG_CRITICAL = eval_context["LOG_CRITICAL"]
params = eval_context["params"]
if not all([params.APP_ID, secrets.APP_SECRET]):
raise UserError(_("Facebook Credentials are not set"))
access_token_app = params.APP_ID + "|" + secrets.APP_SECRET
access_token_page = secrets.PAGE_ACCESS_TOKEN
access_token_user = secrets.USER_ACCESS_TOKEN
def _graph_api(token_type, access_token, method, path, **kwargs):
url = f"https://graph.facebook.com/{params.GRAPH_API_VERSION}/" + path
log_transmission("Facebook Graph API (%s)" % token_type,
"{}\n{}".format(path, kwargs))
kwargs.setdefault("timeout", 5)
if access_token:
kwargs.setdefault("data",
{}).setdefault("access_token", access_token)
response = requests.request(method, url, **kwargs)
log_level = LOG_INFO
try:
data = response.json()
if "error" in data:
log_level = LOG_ERROR
except Exception:
log_level = LOG_CRITICAL
log("Graph API RESPONSE:\n{}".format(response.text), log_level)
return response
def graph_api_app(method, path, **kwargs):
return _graph_api(
"App",
access_token_app,
method,
"%s/%s" % (params.APP_ID, path),
**kwargs,
)
def graph_api_page(method, path, **kwargs):
return _graph_api(
"Page",
access_token_page,
method,
"%s/%s" % (params.PAGE_ID, path),
**kwargs,
)
def graph_api_user(method, path, **kwargs):
return _graph_api(
"User",
access_token_user,
method,
"%s/%s" % (params.USER_ID, path),
**kwargs,
)
# Patch FacebookAdsApi class to add transmission logs
class Api(FacebookAdsApi):
def call(
self,
method,
path,
params=None,
headers=None,
files=None,
url_override=None,
api_version=None,
):
session_name = "unknown session"
if self._session == session_app:
session_name = "App"
elif self._session == session_app:
session_name = "Page"
elif self._session == session_user:
session_name = "User"
log_transmission(
"Facebook (%s)" % session_name,
"{} {}\nparams: {}\nheaders: {}\nfiles: {}\nurl_override: {}\napi_version: {}"
.format(
method,
path,
params,
headers,
files.keys() if files else None,
url_override,
api_version,
),
)
fb_response = super().call(
method,
path,
params=params,
headers=headers,
files=files,
url_override=url_override,
api_version=api_version,
)
log("RESPONSE:\n{}".format(fb_response.json()))
return fb_response
session_app = FacebookSession(
params.APP_ID,
secrets.APP_SECRET,
access_token_app,
)
session_page = FacebookSession(
params.APP_ID,
secrets.APP_SECRET,
access_token_page or
access_token_app, # TODO: if page token is not available, then page api should not be available
)
session_user = FacebookSession(
params.APP_ID,
secrets.APP_SECRET,
access_token_user,
)
api_app = Api(session_app)
api_page = Api(session_page)
api_user = Api(session_user)
FacebookAdsApi.set_default_api(api_app)
def _user2user():
res = _graph_api(
"User",
None,
"GET",
"oauth/access_token",
params={
"grant_type": "fb_exchange_token",
"client_id": params.APP_ID,
"client_secret": secrets.APP_SECRET,
"fb_exchange_token": secrets.USER_ACCESS_TOKEN,
},
)
return res.json()["access_token"]
def user2page():
user_token = _user2user()
res = _graph_api(
"User",
None,
"GET",
"%s" % params.PAGE_ID,
params={
"fields": "access_token",
"access_token": user_token,
},
)
page_token = res.json()["access_token"]
secret_param = self.env["sync.project.secret"].search([
("key", "=", "PAGE_ACCESS_TOKEN"), ("project_id", "=", self.id)
])
if not secret_param:
log("secret PAGE_ACCESS_TOKEN is not found", LOG_WARNING)
else:
secret_param.sudo().write({"value": page_token})
log("secret PAGE_ACCESS_TOKEN is successfully updated")
return {
"FB":
AttrDict(
Application=Application,
Lead=Lead,
),
"FB_TOKEN":
AttrDict(
page=api_page,
user=api_user,
),
"FB_EXCHANGE_TOKEN":
AttrDict(user2page=user2page, ),
"FB_GRAPH_API":
AttrDict(
app=graph_api_app,
page=graph_api_page,
user=graph_api_user,
),
}