def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(
expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
cfps = CampaignFacebookPage.objects.filter(
facebook_id__in=args, first_load=False, deleted=False)
else:
cfps = CampaignFacebookPage.objects.filter(first_load=False,
deleted=False)
error = False
# do first load
for cfp in [c for c in cfps]:
if error:
break
# page info
try:
cfp_info = client.obj_id(cfp.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
else:
if not cfp_info:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
if not cfp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
cfp_info = client.obj_id(cfp.facebook_id)
cfp.name = cfp_info[u'name']
cfp.link = cfp_info[u'link']
cfp.talking_about = cfp_info[u'talking_about_count']
cfp.likes = cfp_info[u'likes']
cfp.save()
evolution = EvolutionFacebookPageLike(
facebook_page=cfp,
likes=cfp_info[u'likes'],
)
evolution.save()
self.stdout.write(
u'Successfully updated Campaign Facebook Page: %s \n\n' %
cfp.facebook_id)
main_loop = True
try:
until = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % cfp.facebook_id,
).offset
except APIPagination.DoesNotExist:
until = None
while main_loop:
if error:
break
params = {
u'limit': 50,
u'format': u'json',
u'method': u'GET',
}
if until:
params.update({u'until': until})
try:
feed = client.feed(cfp.facebook_id, **params)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
else:
if not feed:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
if not feed:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
if not u'data' in feed or not len(feed[u'data']):
main_loop = False
cfp.first_load = True
cfp.save()
continue
page_save = PageSave('metrics_social', 'FacebookMessage')
page_save.start_save(feed[u'data'], fp=cfp)
self.stdout.write(u'Facebook message block was created\n')
self.stdout.write(u'\n'.join(page_save.responses))
if u'paging' in feed and u'next' in feed[u'paging']:
until = urlparse.parse_qs(
urlparse.urlparse(
feed[u'paging'][u'next']).query)[u'until'][0]
self.stdout.write(u'Going to next page \n\n\n')
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % cfp.facebook_id,
)
stopped_at.offset = until
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % cfp.facebook_id,
offset=until,
)
stopped_at.save()
else:
main_loop = False
cfp.first_load = True
cfp.save()
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
bfps = BrandFacebookPage.objects.filter(facebook_id__in=args, first_load=False, deleted=False)
else:
bfps = BrandFacebookPage.objects.filter(first_load=False, deleted=False)
error = False
# do first load
for bfp in [b for b in bfps]:
if error:
break
# page info
try:
bfp_info = client.obj_id(bfp.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
bfp_info = client.obj_id(bfp.facebook_id)
else:
if not bfp_info:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
bfp_info = client.obj_id(bfp.facebook_id)
if not bfp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
bfp_info = client.obj_id(bfp.facebook_id)
bfp.name = bfp_info[u'name']
bfp.link = bfp_info[u'link']
bfp.talking_about = bfp_info[u'talking_about_count']
bfp.likes = bfp_info[u'likes']
bfp.save()
evolution = EvolutionFacebookPageLike(
facebook_page=bfp,
likes=bfp_info[u'likes'],
)
evolution.save()
self.stdout.write(u'Successfully updated Brand Facebook Page: %s \n\n' % bfp.facebook_id)
main_loop = True
try:
until = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % bfp.facebook_id,
).offset
except APIPagination.DoesNotExist:
until = None
while main_loop:
if error:
break
params = {
u'limit': 50,
u'format': u'json',
u'method': u'GET',
}
if until:
params.update({u'until': until})
try:
feed = client.feed(bfp.facebook_id, **params)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
feed = client.feed(bfp.facebook_id, **params)
else:
if not feed:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
feed = client.feed(bfp.facebook_id, **params)
if not feed:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
if not u'data' in feed or not len(feed[u'data']):
main_loop = False
bfp.first_load = True
bfp.save()
continue
page_save = PageSave('metrics_social', 'FacebookMessage')
page_save.start_save(feed[u'data'], fp=bfp)
self.stdout.write(u'Facebook message block was created\n')
self.stdout.write(u'\n'.join(page_save.responses))
if u'paging' in feed and u'next' in feed[u'paging']:
until = urlparse.parse_qs(urlparse.urlparse(feed[u'paging'][u'next']).query)[u'until'][0]
self.stdout.write(u'Going to next page \n\n\n')
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % bfp.facebook_id,
)
stopped_at.offset = until
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='FacebookMessage',
path=u'/%s/feed' % bfp.facebook_id,
offset=until,
)
stopped_at.save()
else:
main_loop = False
bfp.first_load = True
bfp.save()
def handle(self, *args, **options):
try:
ytc = YoutubeConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no YoutubeConnection')
ytc = None
if ytc:
storage = Storage(YoutubeConnection, 'email', ytc.email,
'credential')
credential = storage.get()
if credential is None or credential.invalid == True:
CommandError(u'YoutubeConnection is invalid')
http = httplib2.Http()
http = credential.authorize(http)
service = build('youtube', 'v3', http=http)
if args:
aycs = AdvertiserYoutubeChannel.objects.filter(
youtube_id__in=args, first_load=False, deleted=False)
else:
aycs = AdvertiserYoutubeChannel.objects.filter(
first_load=False, deleted=False)
# do first load
for ayc in [a for a in aycs]:
channels = service.channels().list(
part='statistics', id=ayc.youtube_id).execute()
ayc.views_count = channels[u'items'][0][u'statistics'][
u'viewCount']
ayc.videos_count = channels[u'items'][0][u'statistics'][
u'videoCount']
ayc.subscribers_count = channels[u'items'][0][u'statistics'][
u'subscriberCount']
ayc.save()
evolution = EvolutionYoutubeChannel(
youtube_channel=ayc,
views_count=channels[u'items'][0][u'statistics']
[u'viewCount'],
videos_count=channels[u'items'][0][u'statistics']
[u'videoCount'],
subscribers_count=channels[u'items'][0][u'statistics']
[u'subscriberCount'],
)
evolution.save()
self.stdout.write(
u'Successfully updated Advertiser Youtube Channel: %s \n\n'
% ayc.youtube_id)
main_loop = True
try:
page_token = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='YoutubeVideo',
path=u'/%s/video' % ayc.youtube_id,
).offset
except APIPagination.DoesNotExist:
page_token = None
while main_loop:
search = service.search().list(
part='id',
channelId=ayc.youtube_id,
maxResults=50,
order='date',
pageToken=page_token,
type='video',
).execute()
items = [
item[u'id'][u'videoId'] for item in search[u'items']
if item[u'kind'] == u'youtube#searchResult'
and item[u'id'][u'kind'] == u'youtube#video'
]
videos = service.videos().list(
part='id,snippet,statistics',
id=','.join(items),
maxResults=50,
).execute()
page_save = PageSave('metrics_social', 'YoutubeVideo')
page_save.start_save(videos[u'items'], yc=ayc)
if u'nextPageToken' in search:
page_token = search[u'nextPageToken']
self.stdout.write(u'Going to next page \n\n\n')
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='YoutubeVideo',
path=u'/%s/video' % ayc.youtube_id,
)
stopped_at.offset = page_token
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='YoutubeVideo',
path=u'/%s/video' % ayc.youtube_id,
offset=page_token,
)
stopped_at.save()
else:
main_loop = False
ayc.first_load = True
ayc.save()
def handle(self, *args, **options):
try:
twc = TwitterConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no TwitterConnection')
twc = None
if twc:
# api client
auth = tweepy.OAuthHandler(
settings.TW_CONSUMER_KEY,
settings.TW_CONSUMER_SECRET,
)
auth.set_access_token(twc.access_token, twc.access_token_secret)
api = tweepy.API(auth)
if args:
ctps = CampaignTwitterProfile.objects.filter(twitter_id__in=args, first_load=False, deleted=False)
else:
ctps = CampaignTwitterProfile.objects.filter(first_load=False, deleted=False)
error = False
# do first load
for ctp in [c for c in ctps]:
profile = api.get_user(id=ctp.twitter_id)
ctp.name = profile.name
ctp.screen_name = profile.screen_name
ctp.followers_count = profile.followers_count
ctp.friends_count = profile.friends_count
ctp.listed_count = profile.listed_count
ctp.favourites_count = profile.favourites_count
ctp.statuses_count = profile.statuses_count
ctp.save()
evolution = EvolutionTwitterProfile(
twitter_profile=ctp,
followers_count=ctp.followers_count,
friends_count=ctp.friends_count,
listed_count=ctp.listed_count,
favourites_count=ctp.favourites_count,
statuses_count=ctp.statuses_count,
)
evolution.save()
self.stdout.write(u'Successfully updated Campaign Twitter Profile: %s \n\n' % ctp.twitter_id)
try:
page = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='TwitterMessage',
path=u'/%s/timeline' % ctp.twitter_id,
).offset
page = int(page)
except APIPagination.DoesNotExist:
page = 1
main_loop = True
while main_loop:
statuses = api.user_timeline(user_id=ctp.twitter_id, page=page, count=200)
if statuses:
for status in statuses:
try:
twm = TwitterMessage.objects.get(
twitter_profile=ctp,
twitter_id__exact=status.id_str,
deleted=False,
)
except TwitterMessage.DoesNotExist:
try:
twm = TwitterMessage(
twitter_profile=ctp,
twitter_id=status.id_str,
author_twitter_id=status.author.id_str,
message=status.text,
created_time=status.created_at,
favorites=status.favorite_count,
retweets=status.retweet_count,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='TwitterMessage',
error=u'%s: %s' % (Exception, str(e)),
response=status.id_str,
)
err.save()
self.stdout.write(u'Inserted error message: %s %s' % (Exception, str(e)))
else:
twm.save()
self.stdout.write(u'Inserted message %s' % twm.twitter_id)
else:
self.stdout.write(u'Message already exists: %s' % twm.twitter_id)
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='TwitterMessage',
path=u'/%s/timeline' % ctp.twitter_id,
)
stopped_at.offset = page
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='TwitterMessage',
path=u'/%s/timeline' % ctp.twitter_id,
offset=page,
)
stopped_at.save()
page += 1 # next page
self.stdout.write(u'Going to next page: %s\n\n' % page)
else:
main_loop = False
ctp.first_load = True
ctp.save()
def handle(self, *args, **options):
try:
ytc = YoutubeConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no YoutubeConnection')
ytc = None
if ytc:
storage = Storage(YoutubeConnection, 'email', ytc.email, 'credential')
credential = storage.get()
if credential is None or credential.invalid == True:
CommandError(u'YoutubeConnection is invalid')
http = httplib2.Http()
http = credential.authorize(http)
service = build('youtube', 'v3', http=http)
if args:
bycs = BrandYoutubeChannel.objects.filter(youtube_id__in=args, first_load=False, deleted=False)
else:
bycs = BrandYoutubeChannel.objects.filter(first_load=False, deleted=False)
# do first load
for byc in [b for b in bycs]:
channels = service.channels().list(part='statistics', id=byc.youtube_id).execute()
byc.views_count = channels[u'items'][0][u'statistics'][u'viewCount']
byc.videos_count = channels[u'items'][0][u'statistics'][u'videoCount']
byc.subscribers_count = channels[u'items'][0][u'statistics'][u'subscriberCount']
byc.save()
evolution = EvolutionYoutubeChannel(
youtube_channel=byc,
views_count=channels[u'items'][0][u'statistics'][u'viewCount'],
videos_count=channels[u'items'][0][u'statistics'][u'videoCount'],
subscribers_count=channels[u'items'][0][u'statistics'][u'subscriberCount'],
)
evolution.save()
self.stdout.write(u'Successfully updated Brand Youtube Channel: %s \n\n' % byc.youtube_id)
main_loop = True
try:
page_token = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='YoutubeVideo',
path=u'/%s/video' % byc.youtube_id,
).offset
except APIPagination.DoesNotExist:
page_token = None
while main_loop:
search = service.search().list(
part='id',
channelId=byc.youtube_id,
maxResults=50,
order='date',
pageToken=page_token,
type='video',
).execute()
items = [item[u'id'][u'videoId'] for item in search[u'items']
if item[u'kind'] == u'youtube#searchResult' and item[u'id'][u'kind'] == u'youtube#video']
videos = service.videos().list(
part='id,snippet,statistics',
id=','.join(items),
maxResults=50,
).execute()
page_save = PageSave('metrics_social', 'YoutubeVideo')
page_save.start_save(videos[u'items'], yc=byc)
if u'nextPageToken' in search:
page_token = search[u'nextPageToken']
self.stdout.write(u'Going to next page \n\n\n')
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='YoutubeVideo',
path=u'/%s/video' % byc.youtube_id,
)
stopped_at.offset = page_token
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='YoutubeVideo',
path=u'/%s/video' % byc.youtube_id,
offset=page_token,
)
stopped_at.save()
else:
main_loop = False
byc.first_load = True
byc.save()
def handle(self, *args, **options):
try:
twc = TwitterConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no TwitterConnection')
twc = None
if twc:
# api client
auth = tweepy.OAuthHandler(
settings.TW_CONSUMER_KEY,
settings.TW_CONSUMER_SECRET,
)
auth.set_access_token(twc.access_token, twc.access_token_secret)
api = tweepy.API(auth)
if args:
atps = AdvertiserTwitterProfile.objects.filter(
twitter_id__in=args, first_load=False, deleted=False)
else:
atps = AdvertiserTwitterProfile.objects.filter(
first_load=False, deleted=False)
error = False
# do first load
for atp in [a for a in atps]:
profile = api.get_user(id=atp.twitter_id)
atp.name = profile.name
atp.screen_name = profile.screen_name
atp.followers_count = profile.followers_count
atp.friends_count = profile.friends_count
atp.listed_count = profile.listed_count
atp.favourites_count = profile.favourites_count
atp.statuses_count = profile.statuses_count
atp.save()
evolution = EvolutionTwitterProfile(
twitter_profile=atp,
followers_count=atp.followers_count,
friends_count=atp.friends_count,
listed_count=atp.listed_count,
favourites_count=atp.favourites_count,
statuses_count=atp.statuses_count,
)
evolution.save()
self.stdout.write(
u'Successfully updated Advertiser Twitter Profile: %s \n\n'
% atp.twitter_id)
try:
page = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='TwitterMessage',
path=u'/%s/timeline' % atp.twitter_id,
).offset
page = int(page)
except APIPagination.DoesNotExist:
page = 1
main_loop = True
while main_loop:
statuses = api.user_timeline(user_id=atp.twitter_id,
page=page,
count=200)
if statuses:
for status in statuses:
try:
twm = TwitterMessage.objects.get(
twitter_profile=atp,
twitter_id__exact=status.id_str,
deleted=False,
)
except TwitterMessage.DoesNotExist:
try:
twm = TwitterMessage(
twitter_profile=atp,
twitter_id=status.id_str,
author_twitter_id=status.author.id_str,
message=status.text,
created_time=status.created_at,
favorites=status.favorite_count,
retweets=status.retweet_count,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='TwitterMessage',
error=u'%s: %s' % (Exception, str(e)),
response=status.id_str,
)
err.save()
self.stdout.write(
u'Inserted error message: %s %s' %
(Exception, str(e)))
else:
twm.save()
self.stdout.write(u'Inserted message %s' %
twm.twitter_id)
else:
self.stdout.write(
u'Message already exists: %s' %
twm.twitter_id)
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='TwitterMessage',
path=u'/%s/timeline' % atp.twitter_id,
)
stopped_at.offset = page
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='TwitterMessage',
path=u'/%s/timeline' % atp.twitter_id,
offset=page,
)
stopped_at.save()
page += 1 # next page
self.stdout.write(u'Going to next page: %s\n\n' % page)
else:
main_loop = False
atp.first_load = True
atp.save()