def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(
expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
cfps = CampaignFacebookPage.objects.filter(
facebook_id__in=args, deleted=False)
else:
cfps = CampaignFacebookPage.objects.filter(deleted=False)
error = False
# do daily load
for cfp in [c for c in cfps]:
if error:
break
# page info
try:
cfp_info = client.obj_id(cfp.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
else:
if not cfp_info:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
if not cfp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
cfp.name = cfp_info[u'name']
cfp.link = cfp_info[u'link']
cfp.talking_about = cfp_info[u'talking_about_count']
cfp.likes = cfp_info[u'likes']
cfp.save()
evolution = EvolutionFacebookPageLike(
facebook_page=cfp,
likes=cfp_info[u'likes'],
)
evolution.save()
self.stdout.write(
u'Successfully updated Campaign Facebook Page: %s \n\n' %
cfp.facebook_id)
main_loop = True
until = None
while main_loop:
if error:
break
params = {u'limit': 50}
if until:
params.update({u'until': until})
# load feed
try:
feed = client.feed(cfp.facebook_id, **params)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
else:
if not feed:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
if not feed:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
if not u'data' in feed or not len(feed[u'data']):
main_loop = False
cfp.save()
continue
for row in feed[u'data']:
try:
fbm = FacebookMessage.objects.get(
facebook_id__exact=row[u'id'], deleted=False)
except FacebookMessage.DoesNotExist:
try:
if u'likes' in row and u'count' in row[
u'likes']:
likes = row[u'likes'][u'count']
else:
likes = 0
if u'comments' in row and u'count' in row[
u'comments']:
comments = row[u'comments'][u'count']
else:
comments = 0
content = u''
if u'message' in row:
content = u'%s' % row[u'message']
if u'story' in row:
content = u'%s %s' % (content,
row[u'story'])
if u'picture' in row:
content = u'%s %s' % (content,
row[u'picture'])
if u'link' in row:
content = u'%s %s' % (content,
row[u'link'])
fbm = FacebookMessage(
facebook_page=cfp,
facebook_id=row[u'id'],
author_facebook_id=row[u'from'][u'id'],
message=content,
created_time=datetime.datetime.strptime(
row[u'created_time'],
'%Y-%m-%dT%H:%M:%S+0000'),
message_type=row[u'type'],
likes=0,
comments=0,
shares=row[u'shares'][u'count']
if u'shares' in row else 0,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='FacebookMessage',
error=u'%s: %s' % (Exception, str(e)),
response=row,
)
err.save()
self.stdout.write(
u'Inserted error message: %s %s' %
(Exception, str(e)))
else:
fbm.save()
self.stdout.write(u'Inserted message %s' %
fbm.facebook_id)
else:
self.stdout.write(u'Message already exists: %s' %
fbm.facebook_id)
self.stdout.write(u'Stopping...')
main_loop = False
break
if u'paging' in feed and u'next' in feed[u'paging']:
until = urlparse.parse_qs(
urlparse.urlparse(
feed[u'paging'][u'next']).query)[u'until'][0]
self.stdout.write(u'Going to next page \n\n\n')
else:
main_loop = False
cfp.save()
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
metrics = [
u'post_stories',
u'post_storytellers',
u'post_impressions',
u'post_impressions_unique',
u'post_impressions_paid',
u'post_impressions_paid_unique',
u'post_impressions_fan',
u'post_impressions_fan_unique',
u'post_impressions_fan_paid',
u'post_impressions_fan_paid_unique',
u'post_impressions_organic',
u'post_impressions_organic_unique',
u'post_impressions_viral',
u'post_impressions_viral_unique',
u'post_consumptions',
u'post_consumptions_unique',
u'post_engaged_users',
u'post_negative_feedback',
u'post_negative_feedback_unique',
]
fields = OrderedDict([
(u'facebook_message_id', u'INT'),
(u'facebook_page_id', u'INT'),
(u'metric', u'NVARCHAR(255)'),
(u'period', u'NVARCHAR(255)'),
(u'value', u'BIGINT'),
(u'title', u'NVARCHAR(255)'),
(u'description', u'NVARCHAR(MAX)'),
])
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
# do first load
if args:
ifps = [i for i in InsightsFacebookPage.objects.filter(
facebook_page__facebook_id__in=args,
deleted=False)]
else:
ifps = [i for i in InsightsFacebookPage.objects.filter(deleted=False)]
error = False
for ifp in ifps:
if error:
break
# page info
try:
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
else:
if not ifp_info:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
if not ifp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
ifp.facebook_page.name = ifp_info[u'name']
ifp.facebook_page.link = ifp_info[u'link']
ifp.facebook_page.talking_about = ifp_info[u'talking_about_count']
ifp.facebook_page.likes = ifp_info[u'likes']
ifp.facebook_page.save()
evolution = EvolutionFacebookPageLike(
facebook_page=ifp.facebook_page,
likes=ifp_info[u'likes'],
)
evolution.save()
self.stdout.write(u'Successfully updated Facebook Page: %s \n\n' % ifp.facebook_page.facebook_id)
already_exists = [x for x in InsightsMetric.objects.values_list(
'facebook_message_id',
flat=True,
).filter(
facebook_page=ifp,
metric__in=metrics,
dimension=None,
deleted=False,
) if x]
end_date = datetime.date.today()
end_date = datetime.date.fromordinal(end_date.toordinal() - 2)
fbmsgs = [y for y in FacebookMessage.objects.filter(
facebook_page=ifp.facebook_page,
author_facebook_id=ifp.facebook_page.facebook_id,
created_time__lte=datetime.datetime.combine(end_date, datetime.time()),
deleted=False,
) if y.id not in already_exists or \
y.created_time > datetime.datetime.combine(
datetime.date.fromordinal(end_date.toordinal() - 3), datetime.time()
)]
for fbmsg in fbmsgs:
if error:
break
fbmsgs_tasks = []
for metric in metrics:
path = u'/%s/insights/%s/lifetime' % (fbmsg.facebook_id, metric)
fbmsgs_tasks.append({
u'id': path,
u'path': path,
u'args': [],
u'kwargs': {},
})
try:
me = client.me()
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
else:
if not me:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
client.start_async_tasks(fbmsgs_tasks, fbc=fbc)
if not all([u'response' in task for task in client.tasks]):
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
rows = []
for task in client.tasks:
if u'response' in task:
if not u'data' in task[u'response']:
continue
for data in task[u'response'][u'data']:
for row in data[u'values']:
rows.append((
fbmsg.id,
ifp.id,
str(data[u'name']),
str(data[u'period']),
row[u'value'],
str(data[u'title']),
str(data[u'description'])
))
if not len(task[u'response'][u'data']):
rows.append((
fbmsg.id,
ifp.id,
str(task[u'path'].split('/')[3]),
str(u'Not provided'),
0,
str(u'Not provided'),
str(u'Not provided')
))
if rows:
bk = BulkInsertOrUpdate(fields=fields)
matches = [{
u'clause': None,
u'first_field': u'facebook_message_id',
u'comparison': u'=',
u'second_field': u'facebook_message_id'
},{
u'clause': u'AND',
u'first_field': u'facebook_page_id',
u'comparison': u'=',
u'second_field': u'facebook_page_id'
},{
u'clause': u'AND',
u'first_field': u'metric',
u'comparison': u'=',
u'second_field': u'metric'
},{
u'clause': u'AND',
u'first_field': u'period',
u'comparison': u'=',
u'second_field': u'period'
},]
insert_fields = [
u'facebook_message_id',
u'facebook_page_id',
u'metric',
u'period',
u'value',
u'title',
u'description'
]
cursor = connection.cursor()
cursor.execute(
bk.bulk(
u'SET QUOTED_IDENTIFIER OFF;',
bk.command_bulk_create(),
bk.command_bulk_insert(rows),
bk.command_bulk_merge(
u'metrics_social_insightsmetric',
matches,
[u'value'],
insert_fields
)
)
)
cursor.execute(bk.command_drop_action_table())
transaction.commit_unless_managed()
self.stdout.write(u'Facebook insights block was created\n')
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(
expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
cfps = CampaignFacebookPage.objects.filter(
facebook_id__in=args, first_load=False, deleted=False)
else:
cfps = CampaignFacebookPage.objects.filter(first_load=False,
deleted=False)
error = False
# do first load
for cfp in [c for c in cfps]:
if error:
break
# page info
try:
cfp_info = client.obj_id(cfp.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
else:
if not cfp_info:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
if not cfp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
cfp_info = client.obj_id(cfp.facebook_id)
cfp.name = cfp_info[u'name']
cfp.link = cfp_info[u'link']
cfp.talking_about = cfp_info[u'talking_about_count']
cfp.likes = cfp_info[u'likes']
cfp.save()
evolution = EvolutionFacebookPageLike(
facebook_page=cfp,
likes=cfp_info[u'likes'],
)
evolution.save()
self.stdout.write(
u'Successfully updated Campaign Facebook Page: %s \n\n' %
cfp.facebook_id)
main_loop = True
try:
until = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % cfp.facebook_id,
).offset
except APIPagination.DoesNotExist:
until = None
while main_loop:
if error:
break
params = {
u'limit': 50,
u'format': u'json',
u'method': u'GET',
}
if until:
params.update({u'until': until})
try:
feed = client.feed(cfp.facebook_id, **params)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
else:
if not feed:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
if not feed:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
if not u'data' in feed or not len(feed[u'data']):
main_loop = False
cfp.first_load = True
cfp.save()
continue
page_save = PageSave('metrics_social', 'FacebookMessage')
page_save.start_save(feed[u'data'], fp=cfp)
self.stdout.write(u'Facebook message block was created\n')
self.stdout.write(u'\n'.join(page_save.responses))
if u'paging' in feed and u'next' in feed[u'paging']:
until = urlparse.parse_qs(
urlparse.urlparse(
feed[u'paging'][u'next']).query)[u'until'][0]
self.stdout.write(u'Going to next page \n\n\n')
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % cfp.facebook_id,
)
stopped_at.offset = until
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % cfp.facebook_id,
offset=until,
)
stopped_at.save()
else:
main_loop = False
cfp.first_load = True
cfp.save()
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
bfps = BrandFacebookPage.objects.filter(facebook_id__in=args, first_load=True, deleted=False)
else:
bfps = BrandFacebookPage.objects.filter(first_load=True, deleted=False)
error = False
for bfp in [b for b in bfps]:
if error:
break
# setting params to start loop
offset = 0
limit = 100
loop = True
since = datetime.date.today()
since = datetime.date.fromordinal(since.toordinal() - 15)
# start loop
updated = []
while loop:
if error:
break
fbmsgs = FacebookMessage.objects.exclude(
id__in=updated,
).filter(
facebook_page=bfp,
created_time__gte=since,
loaded_comments=True,
deleted=False,
).order_by('-created_time')[offset:limit]
if not fbmsgs:
loop = False
continue
fbmsgs_tasks = []
for fbmsg in [f for f in fbmsgs]:
# get comments
path = u'/%s/comments' % (fbmsg.facebook_id)
# get pagination
try:
after = APIPagination.objects.get(
app_name=u'metrics_social',
model_name=u'FacebookComment',
path=path,
).offset
except APIPagination.DoesNotExist:
after = None
params = {'limit': '500'}
if after:
params.update({'after': after})
fbmsgs_tasks.append({
u'id': fbmsg.facebook_id,
u'path': path,
u'owner': fbmsg,
u'args': [],
u'kwargs': params,
})
try:
me = client.me()
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
else:
if not me:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
client.start_async_tasks(fbmsgs_tasks, fbc=fbc)
if not all([u'response' in task for task in client.tasks]):
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
for task in client.tasks:
if u'response' in task:
# task opts
owner = task[u'owner']
if not u'data' in task[u'response']:
continue
page_save = PageSave('metrics_social', u'FacebookComment')
page_save.start_save(task[u'response'][u'data'], fbmowner=owner)
owner.comments = FacebookComment.objects.filter(message=owner, deleted=False).count()
owner.save()
if len(task[u'response'][u'data']) < int(task[u'kwargs'][u'limit']):
owner.comments = FacebookComment.objects.filter(message=owner, deleted=False).count()
owner.save()
updated.append(owner.id)
continue
if u'paging' in task[u'response']:
after = task[u'response'][u'paging'][u'cursors'][u'after']
stopped_at, created = APIPagination.objects.get_or_create(
app_name=u'metrics_social',
model_name=u'FacebookComment',
path=task[u'path'],
defaults={
'offset': after,
}
)
stopped_at.offset = after
stopped_at.save()
self.stdout.write(u'Saved pagination for %s\n' % task[u'path'])
if len(fbmsgs) < limit:
loop = False
else:
offset = offset + limit
self.stdout.write(u'Facebook comment block was updated\n')
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
bfps = BrandFacebookPage.objects.filter(facebook_id__in=args, first_load=False, deleted=False)
else:
bfps = BrandFacebookPage.objects.filter(first_load=False, deleted=False)
error = False
# do first load
for bfp in [b for b in bfps]:
if error:
break
# page info
try:
bfp_info = client.obj_id(bfp.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
bfp_info = client.obj_id(bfp.facebook_id)
else:
if not bfp_info:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
bfp_info = client.obj_id(bfp.facebook_id)
if not bfp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
bfp_info = client.obj_id(bfp.facebook_id)
bfp.name = bfp_info[u'name']
bfp.link = bfp_info[u'link']
bfp.talking_about = bfp_info[u'talking_about_count']
bfp.likes = bfp_info[u'likes']
bfp.save()
evolution = EvolutionFacebookPageLike(
facebook_page=bfp,
likes=bfp_info[u'likes'],
)
evolution.save()
self.stdout.write(u'Successfully updated Brand Facebook Page: %s \n\n' % bfp.facebook_id)
main_loop = True
try:
until = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % bfp.facebook_id,
).offset
except APIPagination.DoesNotExist:
until = None
while main_loop:
if error:
break
params = {
u'limit': 50,
u'format': u'json',
u'method': u'GET',
}
if until:
params.update({u'until': until})
try:
feed = client.feed(bfp.facebook_id, **params)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
feed = client.feed(bfp.facebook_id, **params)
else:
if not feed:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
feed = client.feed(bfp.facebook_id, **params)
if not feed:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
if not u'data' in feed or not len(feed[u'data']):
main_loop = False
bfp.first_load = True
bfp.save()
continue
page_save = PageSave('metrics_social', 'FacebookMessage')
page_save.start_save(feed[u'data'], fp=bfp)
self.stdout.write(u'Facebook message block was created\n')
self.stdout.write(u'\n'.join(page_save.responses))
if u'paging' in feed and u'next' in feed[u'paging']:
until = urlparse.parse_qs(urlparse.urlparse(feed[u'paging'][u'next']).query)[u'until'][0]
self.stdout.write(u'Going to next page \n\n\n')
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % bfp.facebook_id,
)
stopped_at.offset = until
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % bfp.facebook_id,
offset=until,
)
stopped_at.save()
else:
main_loop = False
bfp.first_load = True
bfp.save()
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
cfps = CampaignFacebookPage.objects.filter(facebook_id__in=args, first_load=True, deleted=False)
else:
cfps = CampaignFacebookPage.objects.filter(first_load=True, deleted=False)
error = False
for cfp in [c for c in cfps]:
if error:
break
# setting params to start loop
offset = 0
limit = 100
loop = True
since = datetime.date.today()
since = datetime.date.fromordinal(since.toordinal() - 15)
# start loop
updated = []
while loop:
if error:
break
fbmsgs = FacebookMessage.objects.exclude(
id__in=updated,
).filter(
facebook_page=cfp,
created_time__gte=since,
loaded_comments=True,
deleted=False,
).order_by('-created_time')[offset:limit]
if not fbmsgs:
loop = False
continue
fbmsgs_tasks = []
for fbmsg in [f for f in fbmsgs]:
path = u'/%s' % (fbmsg.facebook_id)
params = {u'fields': u'shares'}
fbmsgs_tasks.append({
u'id': fbmsg.facebook_id,
u'path': path,
u'owner': fbmsg,
u'args': [],
u'kwargs': params,
})
try:
me = client.me()
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
else:
if not me:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
client.start_async_tasks(fbmsgs_tasks, fbc=fbc)
if not all([u'response' in task for task in client.tasks]):
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
for task in client.tasks:
if u'response' in task:
# task opts
owner = task[u'owner']
if not u'shares' in task[u'response']:
continue
owner.shares = task[u'response'][u'shares'][u'count']
owner.save()
updated.append(owner.id)
self.stdout.write(u'Facebook message updated\n')
if len(fbmsgs) < limit:
loop = False
else:
offset = offset + limit
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
cfps = CampaignFacebookPage.objects.filter(facebook_id__in=args, deleted=False)
else:
cfps = CampaignFacebookPage.objects.filter(deleted=False)
error = False
# do daily load
for cfp in [c for c in cfps]:
if error:
break
# page info
try:
cfp_info = client.obj_id(cfp.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
else:
if not cfp_info:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
if not cfp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
cfp.name = cfp_info[u'name']
cfp.link = cfp_info[u'link']
cfp.talking_about = cfp_info[u'talking_about_count']
cfp.likes = cfp_info[u'likes']
cfp.save()
evolution = EvolutionFacebookPageLike(
facebook_page=cfp,
likes=cfp_info[u'likes'],
)
evolution.save()
self.stdout.write(u'Successfully updated Campaign Facebook Page: %s \n\n' % cfp.facebook_id)
main_loop = True; until = None
while main_loop:
if error:
break
params = {u'limit': 50}
if until:
params.update({u'until': until})
# load feed
try:
feed = client.feed(cfp.facebook_id, **params)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
else:
if not feed:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
if not feed:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
if not u'data' in feed or not len(feed[u'data']):
main_loop = False
cfp.save()
continue
for row in feed[u'data']:
try:
fbm = FacebookMessage.objects.get(facebook_id__exact=row[u'id'], deleted=False)
except FacebookMessage.DoesNotExist:
try:
if u'likes' in row and u'count' in row[u'likes']:
likes = row[u'likes'][u'count']
else:
likes = 0
if u'comments' in row and u'count' in row[u'comments']:
comments = row[u'comments'][u'count']
else:
comments = 0
content = u''
if u'message' in row:
content = u'%s' % row[u'message']
if u'story' in row:
content = u'%s %s' % (content, row[u'story'])
if u'picture' in row:
content = u'%s %s' % (content, row[u'picture'])
if u'link' in row:
content = u'%s %s' % (content, row[u'link'])
fbm = FacebookMessage(
facebook_page=cfp,
facebook_id=row[u'id'],
author_facebook_id=row[u'from'][u'id'],
message=content,
created_time=datetime.datetime.strptime(row[u'created_time'], '%Y-%m-%dT%H:%M:%S+0000'),
message_type=row[u'type'],
likes=0,
comments=0,
shares=row[u'shares'][u'count'] if u'shares' in row else 0,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='FacebookMessage',
error=u'%s: %s' % (Exception, str(e)),
response=row,
)
err.save()
self.stdout.write(u'Inserted error message: %s %s' % (Exception, str(e)))
else:
fbm.save()
self.stdout.write(u'Inserted message %s' % fbm.facebook_id)
else:
self.stdout.write(u'Message already exists: %s' % fbm.facebook_id)
self.stdout.write(u'Stopping...')
main_loop = False
break
if u'paging' in feed and u'next' in feed[u'paging']:
until = urlparse.parse_qs(urlparse.urlparse(feed[u'paging'][u'next']).query)[u'until'][0]
self.stdout.write(u'Going to next page \n\n\n')
else:
main_loop = False
cfp.save()
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(
expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
metrics = [
u'post_stories',
u'post_storytellers',
u'post_impressions',
u'post_impressions_unique',
u'post_impressions_paid',
u'post_impressions_paid_unique',
u'post_impressions_fan',
u'post_impressions_fan_unique',
u'post_impressions_fan_paid',
u'post_impressions_fan_paid_unique',
u'post_impressions_organic',
u'post_impressions_organic_unique',
u'post_impressions_viral',
u'post_impressions_viral_unique',
u'post_consumptions',
u'post_consumptions_unique',
u'post_engaged_users',
u'post_negative_feedback',
u'post_negative_feedback_unique',
]
fields = OrderedDict([
(u'facebook_message_id', u'INT'),
(u'facebook_page_id', u'INT'),
(u'metric', u'NVARCHAR(255)'),
(u'period', u'NVARCHAR(255)'),
(u'value', u'BIGINT'),
(u'title', u'NVARCHAR(255)'),
(u'description', u'NVARCHAR(MAX)'),
])
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
# do first load
if args:
ifps = [
i for i in FacebookInsightsReload.objects.filter(
facebook_page__facebook_id__in=args,
reload_post_level=False,
deleted=False)
]
else:
ifps = [
i for i in FacebookInsightsReload.objects.filter(
reload_post_level=False, deleted=False)
]
error = False
for ifp in ifps:
if error:
break
# page info
try:
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
else:
if not ifp_info:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
if not ifp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
ifp.facebook_page.name = ifp_info[u'name']
ifp.facebook_page.link = ifp_info[u'link']
ifp.facebook_page.talking_about = ifp_info[
u'talking_about_count']
ifp.facebook_page.likes = ifp_info[u'likes']
ifp.facebook_page.save()
evolution = EvolutionFacebookPageLike(
facebook_page=ifp.facebook_page,
likes=ifp_info[u'likes'],
)
evolution.save()
self.stdout.write(
u'Successfully updated Facebook Page: %s \n\n' %
ifp.facebook_page.facebook_id)
fbmsgs = [
y for y in FacebookMessage.objects.filter(
facebook_page=ifp.facebook_page,
author_facebook_id=ifp.facebook_page.facebook_id,
created_time__lte=datetime.datetime.combine(
ifp.until, datetime.time()),
created_time__gte=datetime.datetime.combine(
ifp.since, datetime.time()),
deleted=False,
) if y.created_time > datetime.datetime.combine(
datetime.date.fromordinal(
datetime.date.today().toordinal() -
3), datetime.time())
]
for fbmsg in fbmsgs:
if error:
break
fbmsgs_tasks = []
for metric in metrics:
path = u'/%s/insights/%s/lifetime' % (
fbmsg.facebook_id, metric)
fbmsgs_tasks.append({
u'id': path,
u'path': path,
u'args': [],
u'kwargs': {},
})
try:
me = client.me()
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
else:
if not me:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
client.start_async_tasks(fbmsgs_tasks, fbc=fbc)
if not all([u'response' in task for task in client.tasks]):
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
rows = []
for task in client.tasks:
if u'response' in task:
if not u'data' in task[u'response']:
continue
for data in task[u'response'][u'data']:
for row in data[u'values']:
rows.append(
(fbmsg.id, ifp.id, str(data[u'name']),
str(data[u'period']), row[u'value'],
str(data[u'title']),
str(data[u'description'])))
if not len(task[u'response'][u'data']):
rows.append((fbmsg.id, ifp.id,
str(task[u'path'].split('/')[3]),
str(u'Not provided'), 0,
str(u'Not provided'),
str(u'Not provided')))
if rows:
bk = BulkInsertOrUpdate(fields=fields)
matches = [
{
u'clause': None,
u'first_field': u'facebook_message_id',
u'comparison': u'=',
u'second_field': u'facebook_message_id'
},
{
u'clause': u'AND',
u'first_field': u'facebook_page_id',
u'comparison': u'=',
u'second_field': u'facebook_page_id'
},
{
u'clause': u'AND',
u'first_field': u'metric',
u'comparison': u'=',
u'second_field': u'metric'
},
{
u'clause': u'AND',
u'first_field': u'period',
u'comparison': u'=',
u'second_field': u'period'
},
]
insert_fields = [
u'facebook_message_id', u'facebook_page_id',
u'metric', u'period', u'value', u'title',
u'description'
]
cursor = connection.cursor()
cursor.execute(
bk.bulk(
u'SET QUOTED_IDENTIFIER OFF;',
bk.command_bulk_create(),
bk.command_bulk_insert(rows),
bk.command_bulk_merge(
u'metrics_social_insightsmetric', matches,
[u'value'], insert_fields)))
cursor.execute(bk.command_drop_action_table())
transaction.commit_unless_managed()
ifp.reload_post_level = True
ifp.save()
self.stdout.write(u'Facebook insights block was created\n')
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
metrics = [
u'page_stories',
u'page_storytellers',
u'page_impressions',
u'page_impressions_unique',
u'page_impressions_paid',
u'page_impressions_paid_unique',
u'page_impressions_organic',
u'page_impressions_organic_unique',
u'page_impressions_viral',
u'page_impressions_viral_unique',
u'page_engaged_users',
u'page_consumptions',
u'page_consumptions_unique',
u'page_negative_feedback',
u'page_negative_feedback_unique',
u'page_views_unique',
u'page_views_login',
u'page_views_login_unique',
u'page_posts_impressions',
u'page_posts_impressions_unique',
u'page_posts_impressions_paid',
u'page_posts_impressions_paid_unique',
u'page_posts_impressions_organic',
u'page_posts_impressions_organic_unique',
u'page_posts_impressions_viral',
u'page_posts_impressions_viral_unique',
]
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
ifps = [i for i in InsightsFacebookPage.objects.filter(
facebook_page__facebook_id__in=args,
first_load_days_28=True,
deleted=False)]
else:
ifps = [i for i in InsightsFacebookPage.objects.filter(first_load_days_28=True, deleted=False)]
error = False
# do first load
for ifp in ifps:
if error:
break
# page info
try:
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
else:
if not ifp_info:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
if not ifp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
ifp.facebook_page.name = ifp_info[u'name']
ifp.facebook_page.link = ifp_info[u'link']
ifp.facebook_page.talking_about = ifp_info[u'talking_about_count']
ifp.facebook_page.likes = ifp_info[u'likes']
ifp.facebook_page.save()
evolution = EvolutionFacebookPageLike(
facebook_page=ifp.facebook_page,
likes=ifp_info[u'likes'],
)
evolution.save()
self.stdout.write(u'Successfully updated Facebook Page: %s \n\n' % ifp.facebook_page.facebook_id)
main_loop = True
lap = 0
until = datetime.date.today()
until = until + relativedelta(days=-2)
since = until + relativedelta(days=-(until.day-1))
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
while main_loop:
if error:
break
insights_tasks = []
for metric in metrics:
path = u'/%s/insights/%s/days_28' % (ifp.facebook_page.facebook_id, metric)
insights_tasks.append({
u'id': path,
u'path': path,
u'args': [],
u'kwargs': {'since': since, 'until': until},
})
try:
me = client.me()
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
else:
if not me:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
me = client.me()
if not me:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
client.start_async_tasks(insights_tasks, fbc=fbc)
if not all([u'response' in task for task in client.tasks]):
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
for task in client.tasks:
if u'response' in task:
if not u'data' in task[u'response']:
continue
for data in task[u'response'][u'data']:
max_days = 0
for row in reversed(data[u'values']):
day = datetime.datetime.strptime(row[u'end_time'][:10], '%Y-%m-%d').date()
day = day + relativedelta(days=-1)
insights_metric, created = InsightsMetric.objects.get_or_create(
facebook_page=ifp,
day=day,
title=data[u'title'],
description=data[u'description'],
metric=data[u'name'],
period=data[u'period'],
deleted=False,
defaults={
'value': row[u'value'],
},
)
if not created:
insights_metric.value = row[u'value']
max_days += 1
insights_metric.save()
if not max_days < 10:
main_loop = False
break
self.stdout.write(u'Facebook insights block was created\n')
responses = [task[u'response'][u'data'][0] for task in client.tasks if len(task[u'response'][u'data'])]
if sum([v[u'value'] for r in responses for v in r[u'values']]):
until = datetime.datetime.fromtimestamp(since).date()
since = until + relativedelta(months=-1, day=1)
until = since + relativedelta(months=+1)
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
self.stdout.write(u'Going to next page \n\n\n')
else:
lap += 1
until = datetime.datetime.fromtimestamp(since).date()
since = until + relativedelta(months=-1, day=1)
until = since + relativedelta(months=+1)
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
if lap > 2:
lap = 0
main_loop = False
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(
expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
afps = AdvertiserFacebookPage.objects.filter(
facebook_id__in=args, first_load=True, deleted=False)
else:
afps = AdvertiserFacebookPage.objects.filter(first_load=True,
deleted=False)
error = False
for afp in [a for a in afps]:
if error:
break
# setting params to start loop
offset = 0
limit = 100
loop = True
since = datetime.date.today()
since = datetime.date.fromordinal(since.toordinal() - 15)
# start loop
updated = []
while loop:
if error:
break
fbmsgs = FacebookMessage.objects.exclude(
id__in=updated, ).filter(
facebook_page=afp,
created_time__gte=since,
loaded_comments=True,
deleted=False,
).order_by('-created_time')[offset:limit]
if not fbmsgs:
loop = False
continue
fbmsgs_tasks = []
for fbmsg in [f for f in fbmsgs]:
path = u'/%s' % (fbmsg.facebook_id)
params = {u'fields': u'shares'}
fbmsgs_tasks.append({
u'id': fbmsg.facebook_id,
u'path': path,
u'owner': fbmsg,
u'args': [],
u'kwargs': params,
})
try:
me = client.me()
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
else:
if not me:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
client.start_async_tasks(fbmsgs_tasks, fbc=fbc)
if not all([u'response' in task for task in client.tasks]):
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
for task in client.tasks:
if u'response' in task:
# task opts
owner = task[u'owner']
if not u'shares' in task[u'response']:
continue
owner.shares = task[u'response'][u'shares'][
u'count']
owner.save()
updated.append(owner.id)
self.stdout.write(u'Facebook message updated\n')
if len(fbmsgs) < limit:
loop = False
else:
offset = offset + limit
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(
expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
metrics = [
u'page_stories',
u'page_storytellers',
u'page_impressions',
u'page_impressions_unique',
u'page_impressions_paid',
u'page_impressions_paid_unique',
u'page_impressions_organic',
u'page_impressions_organic_unique',
u'page_impressions_viral',
u'page_impressions_viral_unique',
u'page_engaged_users',
u'page_negative_feedback',
u'page_negative_feedback_unique',
u'page_fans',
u'page_fans_online_per_day',
u'page_fan_adds',
u'page_fan_removes',
u'page_consumptions',
u'page_consumptions_unique',
u'page_views',
u'page_views_unique',
u'page_views_login',
u'page_views_login_unique',
u'page_posts_impressions',
u'page_posts_impressions_unique',
u'page_posts_impressions_paid',
u'page_posts_impressions_paid_unique',
u'page_posts_impressions_organic',
u'page_posts_impressions_organic_unique',
u'page_posts_impressions_viral',
u'page_posts_impressions_viral_unique',
]
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
ifps = [
i for i in InsightsFacebookPage.objects.filter(
facebook_page__facebook_id__in=args,
first_load_daily=True,
deleted=False)
]
else:
ifps = [
i for i in InsightsFacebookPage.objects.filter(
first_load_daily=True, deleted=False)
]
error = False
# do first load
for ifp in ifps:
if error:
break
# page info
try:
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
else:
if not ifp_info:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
if not ifp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
ifp.facebook_page.name = ifp_info[u'name']
ifp.facebook_page.link = ifp_info[u'link']
ifp.facebook_page.talking_about = ifp_info[
u'talking_about_count']
ifp.facebook_page.likes = ifp_info[u'likes']
ifp.facebook_page.save()
evolution = EvolutionFacebookPageLike(
facebook_page=ifp.facebook_page,
likes=ifp_info[u'likes'],
)
evolution.save()
self.stdout.write(
u'Successfully updated Facebook Page: %s \n\n' %
ifp.facebook_page.facebook_id)
main_loop = True
lap = 0
until = datetime.date.today()
until = until + relativedelta(days=-2)
since = until + relativedelta(days=-(until.day - 1))
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
while main_loop:
if error:
break
insights_tasks = []
for metric in metrics:
if metric == u'page_fans':
path = u'/%s/insights/%s' % (
ifp.facebook_page.facebook_id, metric)
else:
path = u'/%s/insights/%s/day' % (
ifp.facebook_page.facebook_id, metric)
insights_tasks.append({
u'id': path,
u'path': path,
u'args': [],
u'kwargs': {
'since': since,
'until': until
},
})
try:
me = client.me()
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
else:
if not me:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
me = client.me()
if not me:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
client.start_async_tasks(insights_tasks, fbc=fbc)
if not all([u'response' in task for task in client.tasks]):
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
for task in client.tasks:
if u'response' in task:
if not u'data' in task[u'response']:
continue
for data in task[u'response'][u'data']:
max_days = 0
for row in data[u'values']:
day = datetime.datetime.strptime(
row[u'end_time'][:10],
'%Y-%m-%d').date()
day = datetime.date.fromordinal(
day.toordinal() - 1)
insights_metric, created = InsightsMetric.objects.get_or_create(
facebook_page=ifp,
day=day,
title=data[u'title'],
description=data[u'description'],
metric=data[u'name'],
period=data[u'period'],
deleted=False,
defaults={
'value': row[u'value'],
},
)
if not created:
insights_metric.value = row[u'value']
max_days += 1
insights_metric.save()
if not max_days < 10:
main_loop = False
break
self.stdout.write(u'Facebook insights block was created\n')
responses = [
task[u'response'][u'data'][0] for task in client.tasks
if len(task[u'response'][u'data'])
]
if sum(
[v[u'value'] for r in responses
for v in r[u'values']]):
until = datetime.datetime.fromtimestamp(since).date()
since = datetime.date.fromordinal(until.toordinal() -
until.day)
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
self.stdout.write(u'Going to next page \n\n\n')
else:
lap += 1
until = datetime.datetime.fromtimestamp(since).date()
since = datetime.date.fromordinal(until.toordinal() -
until.day)
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
if lap > 2:
lap = 0
main_loop = False
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
metrics = [
u'page_stories',
u'page_storytellers',
u'page_impressions',
u'page_impressions_unique',
u'page_impressions_paid',
u'page_impressions_paid_unique',
u'page_impressions_organic',
u'page_impressions_organic_unique',
u'page_impressions_viral',
u'page_impressions_viral_unique',
u'page_engaged_users',
u'page_consumptions',
u'page_consumptions_unique',
u'page_negative_feedback',
u'page_negative_feedback_unique',
u'page_views_unique',
u'page_views_login',
u'page_views_login_unique',
u'page_posts_impressions',
u'page_posts_impressions_unique',
u'page_posts_impressions_paid',
u'page_posts_impressions_paid_unique',
u'page_posts_impressions_organic',
u'page_posts_impressions_organic_unique',
u'page_posts_impressions_viral',
u'page_posts_impressions_viral_unique',
]
fields = OrderedDict([
(u'facebook_page_id', u'INT'),
(u'day', u'DATETIME'),
(u'metric', u'NVARCHAR(255)'),
(u'period', u'NVARCHAR(255)'),
(u'value', u'BIGINT'),
(u'title', u'NVARCHAR(255)'),
(u'description', u'NVARCHAR(MAX)'),
])
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
ifps = [i for i in FacebookInsightsReload.objects.filter(
facebook_page__facebook_id__in=args,
reload_weekly=False,
deleted=False)]
else:
ifps = [i for i in FacebookInsightsReload.objects.filter(reload_weekly=False, deleted=False)]
error = False
# do first load
for ifp in ifps:
if error:
break
# page info
try:
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
else:
if not ifp_info:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
if not ifp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
ifp.facebook_page.name = ifp_info[u'name']
ifp.facebook_page.link = ifp_info[u'link']
ifp.facebook_page.talking_about = ifp_info[u'talking_about_count']
ifp.facebook_page.likes = ifp_info[u'likes']
ifp.facebook_page.save()
evolution = EvolutionFacebookPageLike(
facebook_page=ifp.facebook_page,
likes=ifp_info[u'likes'],
)
evolution.save()
self.stdout.write(u'Successfully updated Facebook Page: %s \n\n' % ifp.facebook_page.facebook_id)
main_loop = True
until = ifp.until.date()
since = until + relativedelta(days=-(until.day-1))
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
while main_loop:
if error:
break
insights_tasks = []
for metric in metrics:
path = u'/%s/insights/%s/week' % (ifp.facebook_page.facebook_id, metric)
insights_tasks.append({
u'id': path,
u'path': path,
u'args': [],
u'kwargs': {'since': since, 'until': until},
})
try:
me = client.me()
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
else:
if not me:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
me = client.me()
if not me:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
client.start_async_tasks(insights_tasks, fbc=fbc)
if not all([u'response' in task for task in client.tasks]):
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
rows = []
for task in client.tasks:
if u'response' in task:
if not u'data' in task[u'response']:
continue
for data in task[u'response'][u'data']:
for row in reversed(data[u'values']):
day = datetime.datetime.strptime(row[u'end_time'][:10], '%Y-%m-%d').date()
day = day + relativedelta(days=-1)
rows.append((
ifp.id,
str(unicode(day)),
str(data[u'name']),
str(data[u'period']),
row[u'value'],
str(data[u'title']),
str(data[u'description'])
))
if rows:
bk = BulkInsertOrUpdate(fields=fields)
matches = [{
u'clause': None,
u'first_field': u'facebook_page_id',
u'comparison': u'=',
u'second_field': u'facebook_page_id'
},{
u'clause': 'AND',
u'first_field': u'day',
u'comparison': u'=',
u'second_field': u'day'
},{
u'clause': 'AND',
u'first_field': u'metric',
u'comparison': u'=',
u'second_field': u'metric'
},{
u'clause': 'AND',
u'first_field': u'period',
u'comparison': u'=',
u'second_field': u'period'
},]
insert_fields = [
u'facebook_page_id',
u'day',
u'metric',
u'period',
u'value',
u'title',
u'description'
]
cursor = connection.cursor()
cursor.execute(
bk.bulk(
u'SET QUOTED_IDENTIFIER OFF;',
bk.command_bulk_create(),
bk.command_bulk_insert(rows),
bk.command_bulk_merge(
u'metrics_social_insightsmetric',
matches,
[u'value'],
insert_fields
)
)
)
cursor.execute(bk.command_drop_action_table())
transaction.commit_unless_managed()
self.stdout.write(u'Facebook insights block was created\n')
responses = [task[u'response'][u'data'][0] for task in client.tasks if len(task[u'response'][u'data'])]
if datetime.datetime.fromtimestamp(since).date() <= ifp.since.date():
main_loop = False
ifp.reload_weekly = True
ifp.save()
else:
until = datetime.datetime.fromtimestamp(since).date()
since = until + relativedelta(months=-1, day=1)
until = since + relativedelta(months=+1)
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
self.stdout.write(u'Going to next page \n\n\n')
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
metrics = [
u'page_fans_by_like_source_unique',
u'page_stories_by_story_type',
u'page_fans_gender_age', # lifetime
u'page_tab_views_login_top_unique',
u'page_negative_feedback_by_type',
u'page_negative_feedback_by_type_unique',
u'page_fans_online',
]
fields = OrderedDict([
(u'facebook_page_id', u'INT'),
(u'day', u'DATETIME'),
(u'metric', u'NVARCHAR(255)'),
(u'period', u'NVARCHAR(255)'),
(u'value', u'BIGINT'),
(u'dimension', u'NVARCHAR(255)'),
(u'title', u'NVARCHAR(255)'),
(u'description', u'NVARCHAR(MAX)'),
])
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
ifps = [i for i in InsightsFacebookPage.objects.filter(
facebook_page__facebook_id__in=args,
first_load_dimensions=False,
deleted=False)]
else:
ifps = [i for i in InsightsFacebookPage.objects.filter(first_load_dimensions=False, deleted=False)]
error = False
# do first load
for ifp in ifps:
if error:
break
# page info
try:
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
else:
if not ifp_info:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
if not ifp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
ifp.facebook_page.name = ifp_info[u'name']
ifp.facebook_page.link = ifp_info[u'link']
ifp.facebook_page.talking_about = ifp_info[u'talking_about_count']
ifp.facebook_page.likes = ifp_info[u'likes']
ifp.facebook_page.save()
evolution = EvolutionFacebookPageLike(
facebook_page=ifp.facebook_page,
likes=ifp_info[u'likes'],
)
evolution.save()
self.stdout.write(u'Successfully updated Facebook Page: %s \n\n' % ifp.facebook_page.facebook_id)
main_loop = True
lap = 0
try:
since = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='InsightsMetric',
path=u'/%s/insights/page_fans_by_like_source_unique' % ifp.facebook_page.facebook_id,
).offset
since = int(since)
today = datetime.date.today()
until = datetime.datetime.fromtimestamp(since).date()
if until.month == today.month and until.year == today.year:
until = until + relativedelta(days=+(today.day-2))
until = int(time.mktime(until.timetuple()))
else:
until = until + relativedelta(months=+1)
until = until + relativedelta(days=-1)
until = int(time.mktime(until.timetuple()))
except APIPagination.DoesNotExist:
until = datetime.date.today()
until = until + relativedelta(days=-2)
since = until + relativedelta(days=-(until.day-1))
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
while main_loop:
if error:
break
insights_tasks = []
for metric in metrics:
path = u'/%s/insights/%s' % (ifp.facebook_page.facebook_id, metric)
insights_tasks.append({
u'id': path,
u'path': path,
u'args': [],
u'kwargs': {'since': since, 'until': until},
})
try:
me = client.me()
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
else:
if not me:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
me = client.me()
if not me:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
client.start_async_tasks(insights_tasks, fbc=fbc)
if not all([u'response' in task for task in client.tasks]):
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
rows = []
for task in client.tasks:
if u'response' in task:
if not u'data' in task[u'response']:
continue
for data in task[u'response'][u'data']:
for row in reversed(data[u'values']):
day = datetime.datetime.strptime(row[u'end_time'][:10], '%Y-%m-%d').date()
day = day + relativedelta(days=-1)
if type(row[u'value']) is dict:
for k, v in row[u'value'].iteritems():
rows.append((
ifp.id,
str(unicode(day)),
str(data[u'name']),
str(data[u'period']),
v,
str(k),
str(data[u'title']),
str(data[u'description'])
))
if rows:
bk = BulkInsertOrUpdate(fields=fields)
matches = [{
u'clause': None,
u'first_field': u'facebook_page_id',
u'comparison': u'=',
u'second_field': u'facebook_page_id'
},{
u'clause': 'AND',
u'first_field': u'day',
u'comparison': u'=',
u'second_field': u'day'
},{
u'clause': 'AND',
u'first_field': u'metric',
u'comparison': u'=',
u'second_field': u'metric'
},{
u'clause': 'AND',
u'first_field': u'period',
u'comparison': u'=',
u'second_field': u'period'
},{
u'clause': 'AND',
u'first_field': u'dimension',
u'comparison': u'=',
u'second_field': u'dimension'
},]
insert_fields = [
u'facebook_page_id',
u'day',
u'metric',
u'period',
u'value',
u'dimension',
u'title',
u'description'
]
cursor = connection.cursor()
cursor.execute(
bk.bulk(
u'SET QUOTED_IDENTIFIER OFF;',
bk.command_bulk_create(),
bk.command_bulk_insert(rows),
bk.command_bulk_merge(
u'metrics_social_insightsmetric',
matches,
[u'value'],
insert_fields
)
)
)
cursor.execute(bk.command_drop_action_table())
transaction.commit_unless_managed()
self.stdout.write(u'Facebook insights block was created\n')
responses = [task[u'response'][u'data'][0] for task in client.tasks if len(task[u'response'][u'data'])]
if sum([len(v[u'value']) for r in responses for v in r[u'values']]):
until = datetime.datetime.fromtimestamp(since).date()
since = until + relativedelta(months=-1, day=1)
until = since + relativedelta(months=+1)
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
self.stdout.write(u'Going to next page \n\n\n')
stopped_at, created = APIPagination.objects.get_or_create(
app_name=u'metrics_social',
model_name=u'InsightsMetric',
path=u'/%s/insights/page_fans_by_like_source_unique' % ifp.facebook_page.facebook_id,
defaults={
'offset': since,
}
)
stopped_at.offset = since
stopped_at.save()
else:
lap += 1
until = datetime.datetime.fromtimestamp(since).date()
since = until + relativedelta(months=-1, day=1)
until = since + relativedelta(months=+1)
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
if lap > 2:
lap = 0
main_loop = False
ifp.first_load_dimensions = True
ifp.save()
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(
expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
metrics = [
u'page_stories',
u'page_storytellers',
u'page_impressions',
u'page_impressions_unique',
u'page_impressions_paid',
u'page_impressions_paid_unique',
u'page_impressions_organic',
u'page_impressions_organic_unique',
u'page_impressions_viral',
u'page_impressions_viral_unique',
u'page_engaged_users',
u'page_negative_feedback',
u'page_negative_feedback_unique',
u'page_fans',
u'page_fans_online_per_day',
u'page_fan_adds',
u'page_fan_removes',
u'page_consumptions',
u'page_consumptions_unique',
u'page_views',
u'page_views_unique',
u'page_views_login',
u'page_views_login_unique',
u'page_posts_impressions',
u'page_posts_impressions_unique',
u'page_posts_impressions_paid',
u'page_posts_impressions_paid_unique',
u'page_posts_impressions_organic',
u'page_posts_impressions_organic_unique',
u'page_posts_impressions_viral',
u'page_posts_impressions_viral_unique',
]
fields = OrderedDict([
(u'facebook_page_id', u'INT'),
(u'day', u'DATETIME'),
(u'metric', u'NVARCHAR(255)'),
(u'period', u'NVARCHAR(255)'),
(u'value', u'BIGINT'),
(u'title', u'NVARCHAR(255)'),
(u'description', u'NVARCHAR(MAX)'),
])
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
ifps = [
i for i in FacebookInsightsReload.objects.filter(
facebook_page__facebook_id__in=args,
reload_daily=False,
deleted=False)
]
else:
ifps = [
i for i in FacebookInsightsReload.objects.filter(
reload_daily=False, deleted=False)
]
error = False
# do first load
for ifp in ifps:
if error:
break
# page info
try:
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
else:
if not ifp_info:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
ifp_info = client.obj_id(ifp.facebook_page.facebook_id)
if not ifp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
ifp.facebook_page.name = ifp_info[u'name']
ifp.facebook_page.link = ifp_info[u'link']
ifp.facebook_page.talking_about = ifp_info[
u'talking_about_count']
ifp.facebook_page.likes = ifp_info[u'likes']
ifp.facebook_page.save()
evolution = EvolutionFacebookPageLike(
facebook_page=ifp.facebook_page,
likes=ifp_info[u'likes'],
)
evolution.save()
self.stdout.write(
u'Successfully updated Facebook Page: %s \n\n' %
ifp.facebook_page.facebook_id)
main_loop = True
until = ifp.until.date()
since = until + relativedelta(days=-(until.day - 1))
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
while main_loop:
if error:
break
insights_tasks = []
for metric in metrics:
if metric == u'page_fans':
path = u'/%s/insights/%s' % (
ifp.facebook_page.facebook_id, metric)
else:
path = u'/%s/insights/%s/day' % (
ifp.facebook_page.facebook_id, metric)
insights_tasks.append({
u'id': path,
u'path': path,
u'args': [],
u'kwargs': {
'since': since,
'until': until
},
})
try:
me = client.me()
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
else:
if not me:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
me = client.me()
if not me:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
client.start_async_tasks(insights_tasks, fbc=fbc)
if not all([u'response' in task for task in client.tasks]):
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
rows = []
for task in client.tasks:
if u'response' in task:
if not u'data' in task[u'response']:
continue
for data in task[u'response'][u'data']:
for row in reversed(data[u'values']):
day = datetime.datetime.strptime(
row[u'end_time'][:10],
'%Y-%m-%d').date()
day = day + relativedelta(days=-1)
rows.append(
(ifp.id, str(unicode(day)),
str(data[u'name']),
str(data[u'period']), row[u'value'],
str(data[u'title']),
str(data[u'description'])))
if rows:
bk = BulkInsertOrUpdate(fields=fields)
matches = [
{
u'clause': None,
u'first_field': u'facebook_page_id',
u'comparison': u'=',
u'second_field': u'facebook_page_id'
},
{
u'clause': 'AND',
u'first_field': u'day',
u'comparison': u'=',
u'second_field': u'day'
},
{
u'clause': 'AND',
u'first_field': u'metric',
u'comparison': u'=',
u'second_field': u'metric'
},
{
u'clause': 'AND',
u'first_field': u'period',
u'comparison': u'=',
u'second_field': u'period'
},
]
insert_fields = [
u'facebook_page_id', u'day', u'metric', u'period',
u'value', u'title', u'description'
]
cursor = connection.cursor()
cursor.execute(
bk.bulk(
u'SET QUOTED_IDENTIFIER OFF;',
bk.command_bulk_create(),
bk.command_bulk_insert(rows),
bk.command_bulk_merge(
u'metrics_social_insightsmetric', matches,
[u'value'], insert_fields)))
cursor.execute(bk.command_drop_action_table())
transaction.commit_unless_managed()
self.stdout.write(u'Facebook insights block was created\n')
responses = [
task[u'response'][u'data'][0] for task in client.tasks
if len(task[u'response'][u'data'])
]
if datetime.datetime.fromtimestamp(
since).date() <= ifp.since.date():
main_loop = False
ifp.reload_daily = True
ifp.save()
else:
until = datetime.datetime.fromtimestamp(since).date()
since = until + relativedelta(months=-1, day=1)
until = since + relativedelta(months=+1)
since = int(time.mktime(since.timetuple()))
until = int(time.mktime(until.timetuple()))
self.stdout.write(u'Going to next page \n\n\n')