def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(
expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
cfps = CampaignFacebookPage.objects.filter(
facebook_id__in=args, first_load=False, deleted=False)
else:
cfps = CampaignFacebookPage.objects.filter(first_load=False,
deleted=False)
error = False
# do first load
for cfp in [c for c in cfps]:
if error:
break
# page info
try:
cfp_info = client.obj_id(cfp.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
else:
if not cfp_info:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
if not cfp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
cfp_info = client.obj_id(cfp.facebook_id)
cfp.name = cfp_info[u'name']
cfp.link = cfp_info[u'link']
cfp.talking_about = cfp_info[u'talking_about_count']
cfp.likes = cfp_info[u'likes']
cfp.save()
evolution = EvolutionFacebookPageLike(
facebook_page=cfp,
likes=cfp_info[u'likes'],
)
evolution.save()
self.stdout.write(
u'Successfully updated Campaign Facebook Page: %s \n\n' %
cfp.facebook_id)
main_loop = True
try:
until = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % cfp.facebook_id,
).offset
except APIPagination.DoesNotExist:
until = None
while main_loop:
if error:
break
params = {
u'limit': 50,
u'format': u'json',
u'method': u'GET',
}
if until:
params.update({u'until': until})
try:
feed = client.feed(cfp.facebook_id, **params)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
else:
if not feed:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
if not feed:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
if not u'data' in feed or not len(feed[u'data']):
main_loop = False
cfp.first_load = True
cfp.save()
continue
page_save = PageSave('metrics_social', 'FacebookMessage')
page_save.start_save(feed[u'data'], fp=cfp)
self.stdout.write(u'Facebook message block was created\n')
self.stdout.write(u'\n'.join(page_save.responses))
if u'paging' in feed and u'next' in feed[u'paging']:
until = urlparse.parse_qs(
urlparse.urlparse(
feed[u'paging'][u'next']).query)[u'until'][0]
self.stdout.write(u'Going to next page \n\n\n')
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % cfp.facebook_id,
)
stopped_at.offset = until
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % cfp.facebook_id,
offset=until,
)
stopped_at.save()
else:
main_loop = False
cfp.first_load = True
cfp.save()
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
bfps = BrandFacebookPage.objects.filter(facebook_id__in=args, first_load=False, deleted=False)
else:
bfps = BrandFacebookPage.objects.filter(first_load=False, deleted=False)
error = False
# do first load
for bfp in [b for b in bfps]:
if error:
break
# page info
try:
bfp_info = client.obj_id(bfp.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
bfp_info = client.obj_id(bfp.facebook_id)
else:
if not bfp_info:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
bfp_info = client.obj_id(bfp.facebook_id)
if not bfp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
bfp_info = client.obj_id(bfp.facebook_id)
bfp.name = bfp_info[u'name']
bfp.link = bfp_info[u'link']
bfp.talking_about = bfp_info[u'talking_about_count']
bfp.likes = bfp_info[u'likes']
bfp.save()
evolution = EvolutionFacebookPageLike(
facebook_page=bfp,
likes=bfp_info[u'likes'],
)
evolution.save()
self.stdout.write(u'Successfully updated Brand Facebook Page: %s \n\n' % bfp.facebook_id)
main_loop = True
try:
until = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % bfp.facebook_id,
).offset
except APIPagination.DoesNotExist:
until = None
while main_loop:
if error:
break
params = {
u'limit': 50,
u'format': u'json',
u'method': u'GET',
}
if until:
params.update({u'until': until})
try:
feed = client.feed(bfp.facebook_id, **params)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
feed = client.feed(bfp.facebook_id, **params)
else:
if not feed:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
feed = client.feed(bfp.facebook_id, **params)
if not feed:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
if not u'data' in feed or not len(feed[u'data']):
main_loop = False
bfp.first_load = True
bfp.save()
continue
page_save = PageSave('metrics_social', 'FacebookMessage')
page_save.start_save(feed[u'data'], fp=bfp)
self.stdout.write(u'Facebook message block was created\n')
self.stdout.write(u'\n'.join(page_save.responses))
if u'paging' in feed and u'next' in feed[u'paging']:
until = urlparse.parse_qs(urlparse.urlparse(feed[u'paging'][u'next']).query)[u'until'][0]
self.stdout.write(u'Going to next page \n\n\n')
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % bfp.facebook_id,
)
stopped_at.offset = until
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % bfp.facebook_id,
offset=until,
)
stopped_at.save()
else:
main_loop = False
bfp.first_load = True
bfp.save()
def handle(self, *args, **options):
try:
ytc = YoutubeConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no YoutubeConnection')
ytc = None
if ytc:
storage = Storage(YoutubeConnection, 'email', ytc.email,
'credential')
credential = storage.get()
if credential is None or credential.invalid == True:
CommandError(u'YoutubeConnection is invalid')
http = httplib2.Http()
http = credential.authorize(http)
service = build('youtube', 'v3', http=http)
if args:
aycs = AdvertiserYoutubeChannel.objects.filter(
youtube_id__in=args, first_load=False, deleted=False)
else:
aycs = AdvertiserYoutubeChannel.objects.filter(
first_load=False, deleted=False)
# do first load
for ayc in [a for a in aycs]:
channels = service.channels().list(
part='statistics', id=ayc.youtube_id).execute()
ayc.views_count = channels[u'items'][0][u'statistics'][
u'viewCount']
ayc.videos_count = channels[u'items'][0][u'statistics'][
u'videoCount']
ayc.subscribers_count = channels[u'items'][0][u'statistics'][
u'subscriberCount']
ayc.save()
evolution = EvolutionYoutubeChannel(
youtube_channel=ayc,
views_count=channels[u'items'][0][u'statistics']
[u'viewCount'],
videos_count=channels[u'items'][0][u'statistics']
[u'videoCount'],
subscribers_count=channels[u'items'][0][u'statistics']
[u'subscriberCount'],
)
evolution.save()
self.stdout.write(
u'Successfully updated Advertiser Youtube Channel: %s \n\n'
% ayc.youtube_id)
main_loop = True
try:
page_token = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='YoutubeVideo',
path=u'/%s/video' % ayc.youtube_id,
).offset
except APIPagination.DoesNotExist:
page_token = None
while main_loop:
search = service.search().list(
part='id',
channelId=ayc.youtube_id,
maxResults=50,
order='date',
pageToken=page_token,
type='video',
).execute()
items = [
item[u'id'][u'videoId'] for item in search[u'items']
if item[u'kind'] == u'youtube#searchResult'
and item[u'id'][u'kind'] == u'youtube#video'
]
videos = service.videos().list(
part='id,snippet,statistics',
id=','.join(items),
maxResults=50,
).execute()
page_save = PageSave('metrics_social', 'YoutubeVideo')
page_save.start_save(videos[u'items'], yc=ayc)
if u'nextPageToken' in search:
page_token = search[u'nextPageToken']
self.stdout.write(u'Going to next page \n\n\n')
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='YoutubeVideo',
path=u'/%s/video' % ayc.youtube_id,
)
stopped_at.offset = page_token
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='YoutubeVideo',
path=u'/%s/video' % ayc.youtube_id,
offset=page_token,
)
stopped_at.save()
else:
main_loop = False
ayc.first_load = True
ayc.save()
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
bfps = BrandFacebookPage.objects.filter(facebook_id__in=args, first_load=True, deleted=False)
else:
bfps = BrandFacebookPage.objects.filter(first_load=True, deleted=False)
error = False
for bfp in [b for b in bfps]:
if error:
break
# setting params to start loop
offset = 0
limit = 100
loop = True
since = datetime.date.today()
since = datetime.date.fromordinal(since.toordinal() - 15)
# start loop
updated = []
while loop:
if error:
break
fbmsgs = FacebookMessage.objects.exclude(
id__in=updated,
).filter(
facebook_page=bfp,
created_time__gte=since,
loaded_comments=True,
deleted=False,
).order_by('-created_time')[offset:limit]
if not fbmsgs:
loop = False
continue
fbmsgs_tasks = []
for fbmsg in [f for f in fbmsgs]:
# get comments
path = u'/%s/comments' % (fbmsg.facebook_id)
# get pagination
try:
after = APIPagination.objects.get(
app_name=u'metrics_social',
model_name=u'FacebookComment',
path=path,
).offset
except APIPagination.DoesNotExist:
after = None
params = {'limit': '500'}
if after:
params.update({'after': after})
fbmsgs_tasks.append({
u'id': fbmsg.facebook_id,
u'path': path,
u'owner': fbmsg,
u'args': [],
u'kwargs': params,
})
try:
me = client.me()
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
else:
if not me:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
client.start_async_tasks(fbmsgs_tasks, fbc=fbc)
if not all([u'response' in task for task in client.tasks]):
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
for task in client.tasks:
if u'response' in task:
# task opts
owner = task[u'owner']
if not u'data' in task[u'response']:
continue
page_save = PageSave('metrics_social', u'FacebookComment')
page_save.start_save(task[u'response'][u'data'], fbmowner=owner)
owner.comments = FacebookComment.objects.filter(message=owner, deleted=False).count()
owner.save()
if len(task[u'response'][u'data']) < int(task[u'kwargs'][u'limit']):
owner.comments = FacebookComment.objects.filter(message=owner, deleted=False).count()
owner.save()
updated.append(owner.id)
continue
if u'paging' in task[u'response']:
after = task[u'response'][u'paging'][u'cursors'][u'after']
stopped_at, created = APIPagination.objects.get_or_create(
app_name=u'metrics_social',
model_name=u'FacebookComment',
path=task[u'path'],
defaults={
'offset': after,
}
)
stopped_at.offset = after
stopped_at.save()
self.stdout.write(u'Saved pagination for %s\n' % task[u'path'])
if len(fbmsgs) < limit:
loop = False
else:
offset = offset + limit
self.stdout.write(u'Facebook comment block was updated\n')
def handle(self, *args, **options):
try:
ytc = YoutubeConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no YoutubeConnection')
ytc = None
if ytc:
storage = Storage(YoutubeConnection, 'email', ytc.email, 'credential')
credential = storage.get()
if credential is None or credential.invalid == True:
CommandError(u'YoutubeConnection is invalid')
http = httplib2.Http()
http = credential.authorize(http)
service = build('youtube', 'v3', http=http)
if args:
bycs = BrandYoutubeChannel.objects.filter(youtube_id__in=args, first_load=False, deleted=False)
else:
bycs = BrandYoutubeChannel.objects.filter(first_load=False, deleted=False)
# do first load
for byc in [b for b in bycs]:
channels = service.channels().list(part='statistics', id=byc.youtube_id).execute()
byc.views_count = channels[u'items'][0][u'statistics'][u'viewCount']
byc.videos_count = channels[u'items'][0][u'statistics'][u'videoCount']
byc.subscribers_count = channels[u'items'][0][u'statistics'][u'subscriberCount']
byc.save()
evolution = EvolutionYoutubeChannel(
youtube_channel=byc,
views_count=channels[u'items'][0][u'statistics'][u'viewCount'],
videos_count=channels[u'items'][0][u'statistics'][u'videoCount'],
subscribers_count=channels[u'items'][0][u'statistics'][u'subscriberCount'],
)
evolution.save()
self.stdout.write(u'Successfully updated Brand Youtube Channel: %s \n\n' % byc.youtube_id)
main_loop = True
try:
page_token = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='YoutubeVideo',
path=u'/%s/video' % byc.youtube_id,
).offset
except APIPagination.DoesNotExist:
page_token = None
while main_loop:
search = service.search().list(
part='id',
channelId=byc.youtube_id,
maxResults=50,
order='date',
pageToken=page_token,
type='video',
).execute()
items = [item[u'id'][u'videoId'] for item in search[u'items']
if item[u'kind'] == u'youtube#searchResult' and item[u'id'][u'kind'] == u'youtube#video']
videos = service.videos().list(
part='id,snippet,statistics',
id=','.join(items),
maxResults=50,
).execute()
page_save = PageSave('metrics_social', 'YoutubeVideo')
page_save.start_save(videos[u'items'], yc=byc)
if u'nextPageToken' in search:
page_token = search[u'nextPageToken']
self.stdout.write(u'Going to next page \n\n\n')
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='YoutubeVideo',
path=u'/%s/video' % byc.youtube_id,
)
stopped_at.offset = page_token
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='YoutubeVideo',
path=u'/%s/video' % byc.youtube_id,
offset=page_token,
)
stopped_at.save()
else:
main_loop = False
byc.first_load = True
byc.save()