def row_save(cls, row, **kwargs):
# saving message
try:
fbc = cls.objects.get(message=kwargs.get('fbmowner'),
facebook_id__exact=row[u'id'],
deleted=False)
except cls.DoesNotExist:
try:
fbc = cls(
facebook_id=row[u'id'],
message=kwargs.get('fbmowner'),
author_facebook_id=row[u'from'][u'id'],
comment=row[u'message'],
created_time=datetime.strptime(row[u'created_time'],
'%Y-%m-%dT%H:%M:%S+0000'),
likes=row[u'like_count'],
user_likes=row[u'user_likes'],
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='FacebookComment',
error=u'%s: %s' % (Exception, e),
response=row,
)
err.save()
return u'Inserted error comment: %s %s' % (Exception, str(e))
else:
fbc.save()
return u'Inserted comment %s' % fbc.facebook_id
def row_save(cls, row, **kwargs):
# saving message
try:
fbl = cls.objects.get(
message=kwargs.get('fbmowner'),
author_facebook_id__exact=row[u'id'],
deleted=False,
)
except cls.DoesNotExist:
try:
fbl = cls(
message=kwargs.get('fbmowner'),
author_facebook_id=row[u'id'],
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='FacebookLike',
error=u'%s: %s' % (Exception, e),
response=row,
)
err.save()
return u'Inserted error like: %s %s' % (Exception, str(e))
else:
fbl.save()
return u'Inserted like of %s in %s' % (fbl.author_facebook_id, fbl.message.facebook_id)
def row_save(cls, row, **kwargs):
# saving message
try:
fbl = cls.objects.get(
message=kwargs.get('fbmowner'),
author_facebook_id__exact=row[u'id'],
deleted=False,
)
except cls.DoesNotExist:
try:
fbl = cls(
message=kwargs.get('fbmowner'),
author_facebook_id=row[u'id'],
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='FacebookLike',
error=u'%s: %s' % (Exception, e),
response=row,
)
err.save()
return u'Inserted error like: %s %s' % (Exception, str(e))
else:
fbl.save()
return u'Inserted like of %s in %s' % (fbl.author_facebook_id,
fbl.message.facebook_id)
def row_save(cls, row, **kwargs):
# saving message
try:
fbc = cls.objects.get(
message=kwargs.get('fbmowner'),
facebook_id__exact=row[u'id'],
deleted=False
)
except cls.DoesNotExist:
try:
fbc = cls(
facebook_id=row[u'id'],
message=kwargs.get('fbmowner'),
author_facebook_id=row[u'from'][u'id'],
comment=row[u'message'],
created_time=datetime.strptime(row[u'created_time'], '%Y-%m-%dT%H:%M:%S+0000'),
likes=row[u'like_count'],
user_likes=row[u'user_likes'],
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='FacebookComment',
error=u'%s: %s' % (Exception, e),
response=row,
)
err.save()
return u'Inserted error comment: %s %s' % (Exception, str(e))
else:
fbc.save()
return u'Inserted comment %s' % fbc.facebook_id
def row_save(cls, row, **kwargs):
# saving message
try:
fbm = cls.objects.get(
facebook_page=kwargs.get('fp'),
facebook_id__exact=row[u'id'],
deleted=False,
)
except cls.DoesNotExist:
try:
if u'likes' in row and u'count' in row[u'likes']:
likes = row[u'likes'][u'count']
else:
likes = 0
if u'comments' in row and u'count' in row[u'comments']:
comments = row[u'comments'][u'count']
else:
comments = 0
content = u''
if u'message' in row:
content = u'%s' % row[u'message']
if u'story' in row:
content = u'%s %s' % (content, row[u'story'])
if u'picture' in row:
content = u'%s %s' % (content, row[u'picture'])
if u'link' in row:
content = u'%s %s' % (content, row[u'link'])
fbm = cls(
facebook_page=kwargs.get('fp'),
facebook_id=row[u'id'],
author_facebook_id=row[u'from'][u'id'],
message=content,
created_time=datetime.strptime(row[u'created_time'],
'%Y-%m-%dT%H:%M:%S+0000'),
message_type=row[u'type'],
likes=likes,
comments=comments,
shares=row[u'shares'][u'count'] if u'shares' in row else 0,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='FacebookMessage',
error=u'%s: %s' % (Exception, str(e)),
response=row,
)
err.save()
return u'Inserted error message: %s %s' % (Exception, str(e))
else:
fbm.save()
return u'Inserted message %s' % fbm.facebook_id
def row_save(cls, row, **kwargs):
# saving message
try:
fbm = cls.objects.get(
facebook_page=kwargs.get('fp'),
facebook_id__exact=row[u'id'],
deleted=False,
)
except cls.DoesNotExist:
try:
if u'likes' in row and u'count' in row[u'likes']:
likes = row[u'likes'][u'count']
else:
likes = 0
if u'comments' in row and u'count' in row[u'comments']:
comments = row[u'comments'][u'count']
else:
comments = 0
content = u''
if u'message' in row:
content = u'%s' % row[u'message']
if u'story' in row:
content = u'%s %s' % (content, row[u'story'])
if u'picture' in row:
content = u'%s %s' % (content, row[u'picture'])
if u'link' in row:
content = u'%s %s' % (content, row[u'link'])
fbm = cls(
facebook_page=kwargs.get('fp'),
facebook_id=row[u'id'],
author_facebook_id=row[u'from'][u'id'],
message=content,
created_time=datetime.strptime(row[u'created_time'], '%Y-%m-%dT%H:%M:%S+0000'),
message_type=row[u'type'],
likes=likes,
comments=comments,
shares=row[u'shares'][u'count'] if u'shares' in row else 0,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='FacebookMessage',
error=u'%s: %s' % (Exception, str(e)),
response=row,
)
err.save()
return u'Inserted error message: %s %s' % (Exception, str(e))
else:
fbm.save()
return u'Inserted message %s' % fbm.facebook_id
def row_save(cls, row, **kwargs):
# saving video
if row[u'kind'] == u'youtube#video':
try:
ytv = cls.objects.get(
youtube_channel=kwargs.get('yc'),
youtube_id__exact=row[u'id'],
deleted=False,
)
except cls.DoesNotExist:
try:
ytv = cls(
youtube_channel=kwargs.get('yc'),
youtube_id=row[u'id'],
title=row[u'snippet'][u'title'],
description=row[u'snippet'][u'description'],
created_time=datetime.strptime(
row[u'snippet'][u'publishedAt'],
'%Y-%m-%dT%H:%M:%S.000Z'),
views=row[u'statistics'][u'viewCount'],
likes=row[u'statistics'][u'likeCount'],
dislikes=row[u'statistics'][u'dislikeCount'],
favorites=row[u'statistics'][u'favoriteCount'],
comments=row[u'statistics'][u'commentCount'],
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='YoutubeVideo',
error=u'%s: %s' % (Exception, str(e)),
response=row,
)
err.save()
return u'Inserted error video: %s %s' % (Exception, str(e))
else:
ytv.save()
return u'Inserted video %s' % ytv.youtube_id
else:
return u'Video already exists: %s' % ytv.youtube_id
def row_save(cls, row, **kwargs):
# saving video
if row[u'kind'] == u'youtube#video':
try:
ytv = cls.objects.get(
youtube_channel=kwargs.get('yc'),
youtube_id__exact=row[u'id'],
deleted=False,
)
except cls.DoesNotExist:
try:
ytv = cls(
youtube_channel=kwargs.get('yc'),
youtube_id=row[u'id'],
title=row[u'snippet'][u'title'],
description=row[u'snippet'][u'description'],
created_time=datetime.strptime(row[u'snippet'][u'publishedAt'], '%Y-%m-%dT%H:%M:%S.000Z'),
views=row[u'statistics'][u'viewCount'],
likes=row[u'statistics'][u'likeCount'],
dislikes=row[u'statistics'][u'dislikeCount'],
favorites=row[u'statistics'][u'favoriteCount'],
comments=row[u'statistics'][u'commentCount'],
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='YoutubeVideo',
error=u'%s: %s' % (Exception, str(e)),
response=row,
)
err.save()
return u'Inserted error video: %s %s' % (Exception, str(e))
else:
ytv.save()
return u'Inserted video %s' % ytv.youtube_id
else:
return u'Video already exists: %s' % ytv.youtube_id
def handle(self, *args, **options):
try:
ytc = YoutubeConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no YoutubeConnection')
ytc = None
base_url = u'http://gdata.youtube.com/feeds/api/videos/'
yt_service = gdata.youtube.service.YouTubeService()
if args:
cycs = CampaignYoutubeChannel.objects.filter(yourube_id__in=args, first_load=True, deleted=False)
else:
cycs = CampaignYoutubeChannel.objects.filter(first_load=True, deleted=False)
error = False
for cyc in [c for c in cycs]:
if error:
break
since = datetime.date.today()
since = datetime.date.fromordinal(since.toordinal() - 15)
ytvs = YoutubeVideo.objects.filter(
youtube_channel=cyc,
created_time__gte=since,
loaded_comments=True,
deleted=False,
).order_by('-created_time')
for ytv in [y for y in ytvs]:
# get pagination
try:
start_index = APIPagination.objects.get(
app_name=u'metrics_social',
model_name=u'YoutubeComment',
path=u'/%s/comments' % (ytv.youtube_id),
).offset
except APIPagination.DoesNotExist:
start_index = None
if not start_index:
url = base_url + (u'%s/comments' % ytv.youtube_id)
feed = yt_service.GetYouTubeVideoCommentFeed(uri=url)
else:
params = {
u'start-index': start_index,
u'max-results': '25',
u'direction': 'next',
}
url = base_url + (u'%s/comments' % ytv.youtube_id) + urlencode(params)
feed = yt_service.Query(url)
while feed:
for entry in feed.entry:
_id = entry.id.text.split('/')[-1]
_username = entry.author[0].uri.text.split('/')[-1].decode('utf8')
_title = entry.title.text.decode('utf8')
_comment = entry.content.text.decode('utf8')
_created_time = datetime.datetime.strptime(entry.published.text, '%Y-%m-%dT%H:%M:%S.000Z')
try:
ytc = YoutubeComment.objects.get(
video=ytv,
youtube_id__exact=_id,
deleted=False
)
except YoutubeComment.DoesNotExist:
try:
ytc = YoutubeComment(
youtube_id=_id,
video=ytv,
author_youtube_username=_username,
title=_title,
comment=_comment,
created_time=_created_time,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='YoutubeComment',
error=u'%s: %s' % (Exception, e),
response=entry,
)
err.save()
self.stdout.write(u'Inserted error comment: %s %s' % (Exception, str(e)))
else:
ytc.save()
self.stdout.write(u'Inserted comment %s' % ytc.youtube_id)
else:
self.stdout.write(u'Comment already exists: %s' % ytc.youtube_id)
if feed.GetNextLink():
start_index = urlparse.parse_qs(urlparse.urlparse(feed.GetNextLink().href).query)
stopped_at, created = APIPagination.objects.get_or_create(
app_name=u'metrics_social',
model_name=u'YoutubeComment',
path=u'/%s/comments' % (ytv.youtube_id),
defaults={
'offset': start_index[u'start-index'][0],
}
)
stopped_at.offset = start_index[u'start-index'][0]
stopped_at.save()
self.stdout.write(u'Saved pagination for %s\n' % ytv.youtube_id)
feed = yt_service.Query(feed.GetNextLink().href)
else:
feed = None
if ytc:
storage = Storage(YoutubeConnection, 'email', ytc.email, 'credential')
credential = storage.get()
if credential is None or credential.invalid == True:
CommandError(u'YoutubeConnection is invalid')
http = httplib2.Http()
http = credential.authorize(http)
service = build('youtube', 'v3', http=http)
ids = [y.youtube_id for y in ytvs]
videos = service.videos().list(
part='id,snippet,statistics',
id=','.join(ids),
maxResults=50,
).execute()
for row in videos[u'items']:
if row[u'kind'] == u'youtube#video':
ytv = YoutubeVideo.objects.get(
youtube_id__exact=row[u'id'],
deleted=False,
)
ytv.views = row[u'statistics'][u'viewCount']
ytv.likes = row[u'statistics'][u'likeCount']
ytv.dislikes = row[u'statistics'][u'dislikeCount']
ytv.favorites = row[u'statistics'][u'favoriteCount']
ytv.comments = row[u'statistics'][u'commentCount']
ytv.save()
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
cfps = CampaignFacebookPage.objects.filter(facebook_id__in=args, deleted=False)
else:
cfps = CampaignFacebookPage.objects.filter(deleted=False)
error = False
# do daily load
for cfp in [c for c in cfps]:
if error:
break
# page info
try:
cfp_info = client.obj_id(cfp.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
else:
if not cfp_info:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
if not cfp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
cfp.name = cfp_info[u'name']
cfp.link = cfp_info[u'link']
cfp.talking_about = cfp_info[u'talking_about_count']
cfp.likes = cfp_info[u'likes']
cfp.save()
evolution = EvolutionFacebookPageLike(
facebook_page=cfp,
likes=cfp_info[u'likes'],
)
evolution.save()
self.stdout.write(u'Successfully updated Campaign Facebook Page: %s \n\n' % cfp.facebook_id)
main_loop = True; until = None
while main_loop:
if error:
break
params = {u'limit': 50}
if until:
params.update({u'until': until})
# load feed
try:
feed = client.feed(cfp.facebook_id, **params)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
else:
if not feed:
fbc, access_token = FacebookConnection.renew_client(fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
if not feed:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
if not u'data' in feed or not len(feed[u'data']):
main_loop = False
cfp.save()
continue
for row in feed[u'data']:
try:
fbm = FacebookMessage.objects.get(facebook_id__exact=row[u'id'], deleted=False)
except FacebookMessage.DoesNotExist:
try:
if u'likes' in row and u'count' in row[u'likes']:
likes = row[u'likes'][u'count']
else:
likes = 0
if u'comments' in row and u'count' in row[u'comments']:
comments = row[u'comments'][u'count']
else:
comments = 0
content = u''
if u'message' in row:
content = u'%s' % row[u'message']
if u'story' in row:
content = u'%s %s' % (content, row[u'story'])
if u'picture' in row:
content = u'%s %s' % (content, row[u'picture'])
if u'link' in row:
content = u'%s %s' % (content, row[u'link'])
fbm = FacebookMessage(
facebook_page=cfp,
facebook_id=row[u'id'],
author_facebook_id=row[u'from'][u'id'],
message=content,
created_time=datetime.datetime.strptime(row[u'created_time'], '%Y-%m-%dT%H:%M:%S+0000'),
message_type=row[u'type'],
likes=0,
comments=0,
shares=row[u'shares'][u'count'] if u'shares' in row else 0,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='FacebookMessage',
error=u'%s: %s' % (Exception, str(e)),
response=row,
)
err.save()
self.stdout.write(u'Inserted error message: %s %s' % (Exception, str(e)))
else:
fbm.save()
self.stdout.write(u'Inserted message %s' % fbm.facebook_id)
else:
self.stdout.write(u'Message already exists: %s' % fbm.facebook_id)
self.stdout.write(u'Stopping...')
main_loop = False
break
if u'paging' in feed and u'next' in feed[u'paging']:
until = urlparse.parse_qs(urlparse.urlparse(feed[u'paging'][u'next']).query)[u'until'][0]
self.stdout.write(u'Going to next page \n\n\n')
else:
main_loop = False
cfp.save()
def handle(self, *args, **options):
try:
ytc = YoutubeConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no YoutubeConnection')
ytc = None
if ytc:
storage = Storage(YoutubeConnection, 'email', ytc.email, 'credential')
credential = storage.get()
if credential is None or credential.invalid == True:
CommandError(u'YoutubeConnection is invalid')
http = httplib2.Http()
http = credential.authorize(http)
service = build('youtube', 'v3', http=http)
if args:
cycs = CampaignYoutubeChannel.objects.filter(youtube_id__in=args, first_load=True, deleted=False)
else:
cycs = CampaignYoutubeChannel.objects.filter(first_load=True, deleted=False)
# do first load
for cyc in [c for c in cycs]:
channels = service.channels().list(part='statistics', id=cyc.youtube_id).execute()
if not channels[u'items']:
continue
cyc.views_count = channels[u'items'][0][u'statistics'][u'viewCount']
cyc.videos_count = channels[u'items'][0][u'statistics'][u'videoCount']
cyc.subscribers_count = channels[u'items'][0][u'statistics'][u'subscriberCount']
cyc.save()
evolution = EvolutionYoutubeChannel(
youtube_channel=cyc,
views_count=channels[u'items'][0][u'statistics'][u'viewCount'],
videos_count=channels[u'items'][0][u'statistics'][u'videoCount'],
subscribers_count=channels[u'items'][0][u'statistics'][u'subscriberCount'],
)
evolution.save()
self.stdout.write(u'Successfully updated Campaign Youtube Channel: %s \n\n' % cyc.youtube_id)
main_loop = True
page_token = None
while main_loop:
search = service.search().list(
part='id',
channelId=cyc.youtube_id,
maxResults=50,
order='date',
pageToken=page_token,
type='video',
).execute()
items = [item[u'id'][u'videoId'] for item in search[u'items']
if item[u'kind'] == u'youtube#searchResult' and item[u'id'][u'kind'] == u'youtube#video']
videos = service.videos().list(
part='id,snippet,statistics',
id=','.join(items),
maxResults=50,
).execute()
for row in videos[u'items']:
# saving video
if row[u'kind'] == u'youtube#video':
try:
ytv = YoutubeVideo.objects.get(
youtube_channel=cyc,
youtube_id__exact=row[u'id'],
deleted=False,
)
except YoutubeVideo.DoesNotExist:
try:
ytv = YoutubeVideo(
youtube_channel=cyc,
youtube_id=row[u'id'],
title=row[u'snippet'][u'title'],
description=row[u'snippet'][u'description'],
created_time=datetime.datetime.strptime(row[u'snippet'][u'publishedAt'], '%Y-%m-%dT%H:%M:%S.000Z'),
views=row[u'statistics'][u'viewCount'],
likes=row[u'statistics'][u'likeCount'],
dislikes=row[u'statistics'][u'dislikeCount'],
favorites=row[u'statistics'][u'favoriteCount'],
comments=row[u'statistics'][u'commentCount'],
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='YoutubeVideo',
error=u'%s: %s' % (Exception, str(e)),
response=row,
)
err.save()
self.stdout.write(u'Inserted error video: %s %s' % (Exception, str(e)))
else:
ytv.save()
self.stdout.write(u'Inserted video %s' % ytv.youtube_id)
else:
main_loop = False
self.stdout.write(u'Video already exists: %s' % ytv.youtube_id)
break
if u'nextPageToken' in search:
page_token = search[u'nextPageToken']
else:
main_loop = False
def handle(self, *args, **options):
try:
ytc = YoutubeConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no YoutubeConnection')
ytc = None
base_url = u'http://gdata.youtube.com/feeds/api/videos/'
yt_service = gdata.youtube.service.YouTubeService()
if args:
aycs = AdvertiserYoutubeChannel.objects.filter(yourube_id__in=args,
first_load=True,
deleted=False)
else:
aycs = AdvertiserYoutubeChannel.objects.filter(first_load=True,
deleted=False)
error = False
for ayc in [a for a in aycs]:
if error:
break
since = datetime.date.today()
since = datetime.date.fromordinal(since.toordinal() - 15)
ytvs = YoutubeVideo.objects.filter(
youtube_channel=ayc,
created_time__gte=since,
loaded_comments=True,
deleted=False,
).order_by('-created_time')
for ytv in [y for y in ytvs]:
# get pagination
try:
start_index = APIPagination.objects.get(
app_name=u'metrics_social',
model_name=u'YoutubeComment',
path=u'/%s/comments' % (ytv.youtube_id),
).offset
except APIPagination.DoesNotExist:
start_index = None
if not start_index:
url = base_url + (u'%s/comments' % ytv.youtube_id)
feed = yt_service.GetYouTubeVideoCommentFeed(uri=url)
else:
params = {
u'start-index': start_index,
u'max-results': '25',
u'direction': 'next',
}
url = base_url + (u'%s/comments' %
ytv.youtube_id) + urlencode(params)
feed = yt_service.Query(url)
while feed:
for entry in feed.entry:
_id = entry.id.text.split('/')[-1]
_username = entry.author[0].uri.text.split(
'/')[-1].decode('utf8')
_title = entry.title.text.decode('utf8')
_comment = entry.content.text.decode('utf8')
_created_time = datetime.datetime.strptime(
entry.published.text, '%Y-%m-%dT%H:%M:%S.000Z')
try:
ytc = YoutubeComment.objects.get(
video=ytv,
youtube_id__exact=_id,
deleted=False)
except YoutubeComment.DoesNotExist:
try:
ytc = YoutubeComment(
youtube_id=_id,
video=ytv,
author_youtube_username=_username,
title=_title,
comment=_comment,
created_time=_created_time,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='YoutubeComment',
error=u'%s: %s' % (Exception, e),
response=entry,
)
err.save()
self.stdout.write(
u'Inserted error comment: %s %s' %
(Exception, str(e)))
else:
ytc.save()
self.stdout.write(u'Inserted comment %s' %
ytc.youtube_id)
else:
self.stdout.write(u'Comment already exists: %s' %
ytc.youtube_id)
if feed.GetNextLink():
start_index = urlparse.parse_qs(
urlparse.urlparse(feed.GetNextLink().href).query)
stopped_at, created = APIPagination.objects.get_or_create(
app_name=u'metrics_social',
model_name=u'YoutubeComment',
path=u'/%s/comments' % (ytv.youtube_id),
defaults={
'offset': start_index[u'start-index'][0],
})
stopped_at.offset = start_index[u'start-index'][0]
stopped_at.save()
self.stdout.write(u'Saved pagination for %s\n' %
ytv.youtube_id)
feed = yt_service.Query(feed.GetNextLink().href)
else:
feed = None
if ytc:
storage = Storage(YoutubeConnection, 'email', ytc.email,
'credential')
credential = storage.get()
if credential is None or credential.invalid == True:
CommandError(u'YoutubeConnection is invalid')
http = httplib2.Http()
http = credential.authorize(http)
service = build('youtube', 'v3', http=http)
ids = [y.youtube_id for y in ytvs]
videos = service.videos().list(
part='id,snippet,statistics',
id=','.join(ids),
maxResults=50,
).execute()
for row in videos[u'items']:
if row[u'kind'] == u'youtube#video':
ytv = YoutubeVideo.objects.get(
youtube_id__exact=row[u'id'],
deleted=False,
)
ytv.views = row[u'statistics'][u'viewCount']
ytv.likes = row[u'statistics'][u'likeCount']
ytv.dislikes = row[u'statistics'][u'dislikeCount']
ytv.favorites = row[u'statistics'][u'favoriteCount']
ytv.comments = row[u'statistics'][u'commentCount']
ytv.save()
def handle(self, *args, **options):
try:
ytc = YoutubeConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no YoutubeConnection')
ytc = None
if ytc:
storage = Storage(YoutubeConnection, 'email', ytc.email,
'credential')
credential = storage.get()
if credential is None or credential.invalid == True:
CommandError(u'YoutubeConnection is invalid')
http = httplib2.Http()
http = credential.authorize(http)
service = build('youtube', 'v3', http=http)
if args:
cycs = CampaignYoutubeChannel.objects.filter(
youtube_id__in=args, first_load=True, deleted=False)
else:
cycs = CampaignYoutubeChannel.objects.filter(first_load=True,
deleted=False)
# do first load
for cyc in [c for c in cycs]:
channels = service.channels().list(
part='statistics', id=cyc.youtube_id).execute()
if not channels[u'items']:
continue
cyc.views_count = channels[u'items'][0][u'statistics'][
u'viewCount']
cyc.videos_count = channels[u'items'][0][u'statistics'][
u'videoCount']
cyc.subscribers_count = channels[u'items'][0][u'statistics'][
u'subscriberCount']
cyc.save()
evolution = EvolutionYoutubeChannel(
youtube_channel=cyc,
views_count=channels[u'items'][0][u'statistics']
[u'viewCount'],
videos_count=channels[u'items'][0][u'statistics']
[u'videoCount'],
subscribers_count=channels[u'items'][0][u'statistics']
[u'subscriberCount'],
)
evolution.save()
self.stdout.write(
u'Successfully updated Campaign Youtube Channel: %s \n\n' %
cyc.youtube_id)
main_loop = True
page_token = None
while main_loop:
search = service.search().list(
part='id',
channelId=cyc.youtube_id,
maxResults=50,
order='date',
pageToken=page_token,
type='video',
).execute()
items = [
item[u'id'][u'videoId'] for item in search[u'items']
if item[u'kind'] == u'youtube#searchResult'
and item[u'id'][u'kind'] == u'youtube#video'
]
videos = service.videos().list(
part='id,snippet,statistics',
id=','.join(items),
maxResults=50,
).execute()
for row in videos[u'items']:
# saving video
if row[u'kind'] == u'youtube#video':
try:
ytv = YoutubeVideo.objects.get(
youtube_channel=cyc,
youtube_id__exact=row[u'id'],
deleted=False,
)
except YoutubeVideo.DoesNotExist:
try:
ytv = YoutubeVideo(
youtube_channel=cyc,
youtube_id=row[u'id'],
title=row[u'snippet'][u'title'],
description=row[u'snippet']
[u'description'],
created_time=datetime.datetime.
strptime(
row[u'snippet'][u'publishedAt'],
'%Y-%m-%dT%H:%M:%S.000Z'),
views=row[u'statistics'][u'viewCount'],
likes=row[u'statistics'][u'likeCount'],
dislikes=row[u'statistics']
[u'dislikeCount'],
favorites=row[u'statistics']
[u'favoriteCount'],
comments=row[u'statistics']
[u'commentCount'],
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='YoutubeVideo',
error=u'%s: %s' % (Exception, str(e)),
response=row,
)
err.save()
self.stdout.write(
u'Inserted error video: %s %s' %
(Exception, str(e)))
else:
ytv.save()
self.stdout.write(u'Inserted video %s' %
ytv.youtube_id)
else:
main_loop = False
self.stdout.write(u'Video already exists: %s' %
ytv.youtube_id)
break
if u'nextPageToken' in search:
page_token = search[u'nextPageToken']
else:
main_loop = False
def handle(self, *args, **options):
base_url = u'http://gdata.youtube.com/feeds/api/videos/'
yt_service = gdata.youtube.service.YouTubeService()
if args:
cycs = CampaignYoutubeChannel.objects.filter(youtube_id__in=args, first_load=True, deleted=False)
else:
cycs = CampaignYoutubeChannel.objects.filter(first_load=True, deleted=False)
error = False
for cyc in [c for c in cycs]:
if error:
break
ytvs = YoutubeVideo.objects.filter(
youtube_channel=cyc,
loaded_comments=False,
deleted=False,
).order_by('-created_time')
for ytv in [y for y in ytvs]:
# get pagination
try:
start_index = APIPagination.objects.get(
app_name=u'metrics_social',
model_name=u'YoutubeComment',
path=u'/%s/comments' % (ytv.youtube_id),
).offset
except APIPagination.DoesNotExist:
start_index = None
if not start_index:
url = base_url + (u'%s/comments' % ytv.youtube_id)
try:
feed = yt_service.GetYouTubeVideoCommentFeed(uri=url)
except Exception, e:
ytv.loaded_comments = True
ytv.save()
feed = None
self.stdout.write(u'Error when load feed: %s %s' % (Exception, str(e)))
else:
params = {
u'start-index': start_index,
u'max-results': '25',
u'direction': 'next',
}
url = base_url + (u'%s/comments' % ytv.youtube_id) + urlencode(params)
try:
feed = yt_service.Query(url)
except Exception, e:
ytv.loaded_comments = True
ytv.save()
feed = None
self.stdout.write(u'Error when load feed: %s %s' % (Exception, str(e)))
while feed:
for entry in feed.entry:
_id = entry.id.text.split('/')[-1]
_username = entry.author[0].uri.text.split('/')[-1].decode('utf8')
try:
_title = entry.title.text.decode('utf8')
except Exception, e:
_title = ''
try:
_comment = entry.content.text.decode('utf8')
except Exception, e:
_comment = ''
_created_time = datetime.datetime.strptime(entry.published.text, '%Y-%m-%dT%H:%M:%S.000Z')
try:
ytc = YoutubeComment.objects.get(
video=ytv,
youtube_id__exact=_id,
deleted=False
)
except YoutubeComment.DoesNotExist:
try:
ytc = YoutubeComment(
youtube_id=_id,
video=ytv,
author_youtube_username=_username,
title=_title,
comment=_comment,
created_time=_created_time,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='YoutubeComment',
error=u'%s: %s' % (Exception, e),
response=entry,
)
err.save()
self.stdout.write(u'Inserted error comment: %s %s' % (Exception, str(e)))
else:
ytc.save()
self.stdout.write(u'Inserted comment %s' % ytc.youtube_id)
def handle(self, *args, **options):
base_url = u'http://gdata.youtube.com/feeds/api/videos/'
yt_service = gdata.youtube.service.YouTubeService()
if args:
bycs = BrandYoutubeChannel.objects.filter(youtube_id__in=args, first_load=True, deleted=False)
else:
bycs = BrandYoutubeChannel.objects.filter(first_load=True, deleted=False)
error = False
for byc in [b for b in bycs]:
if error:
break
ytvs = YoutubeVideo.objects.filter(
youtube_channel=byc,
loaded_comments=False,
deleted=False,
).order_by('-created_time')
for ytv in [y for y in ytvs]:
# get pagination
try:
start_index = APIPagination.objects.get(
app_name=u'metrics_social',
model_name=u'YoutubeComment',
path=u'/%s/comments' % (ytv.youtube_id),
).offset
except APIPagination.DoesNotExist:
start_index = None
if not start_index:
url = base_url + (u'%s/comments' % ytv.youtube_id)
try:
feed = yt_service.GetYouTubeVideoCommentFeed(uri=url)
except Exception, e:
ytv.loaded_comments = True
ytv.save()
feed = None
self.stdout.write(u'Error when load feed: %s %s' % (Exception, str(e)))
else:
params = {
u'start-index': start_index,
u'max-results': '25',
u'direction': 'next',
}
url = base_url + (u'%s/comments' % ytv.youtube_id) + urlencode(params)
try:
feed = yt_service.Query(url)
except Exception, e:
ytv.loaded_comments = True
ytv.save()
feed = None
self.stdout.write(u'Error when load feed: %s %s' % (Exception, str(e)))
while feed:
for entry in feed.entry:
_id = entry.id.text.split('/')[-1]
_username = entry.author[0].uri.text.split('/')[-1].decode('utf8')
try:
_title = entry.title.text.decode('utf8')
except Exception, e:
_title = ''
try:
_comment = entry.content.text.decode('utf8')
except Exception, e:
_comment = ''
_created_time = datetime.datetime.strptime(entry.published.text, '%Y-%m-%dT%H:%M:%S.000Z')
try:
ytc = YoutubeComment.objects.get(
video=ytv,
youtube_id__exact=_id,
deleted=False
)
except YoutubeComment.DoesNotExist:
try:
ytc = YoutubeComment(
youtube_id=_id,
video=ytv,
author_youtube_username=_username,
title=_title,
comment=_comment,
created_time=_created_time,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='YoutubeComment',
error=u'%s: %s' % (Exception, e),
response=entry,
)
err.save()
self.stdout.write(u'Inserted error comment: %s %s' % (Exception, str(e)))
else:
ytc.save()
self.stdout.write(u'Inserted comment %s' % ytc.youtube_id)
def handle(self, *args, **options):
try:
twc = TwitterConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no TwitterConnection')
twc = None
if twc:
# api client
auth = tweepy.OAuthHandler(
settings.TW_CONSUMER_KEY,
settings.TW_CONSUMER_SECRET,
)
auth.set_access_token(twc.access_token, twc.access_token_secret)
api = tweepy.API(auth)
if args:
ctps = CampaignTwitterProfile.objects.filter(twitter_id__in=args, first_load=True, deleted=False)
else:
ctps = CampaignTwitterProfile.objects.filter(first_load=True, deleted=False)
error = False
# do first load
for ctp in [c for c in ctps]:
profile = api.get_user(id=ctp.twitter_id)
ctp.name = profile.name
ctp.screen_name = profile.screen_name
ctp.followers_count = profile.followers_count
ctp.friends_count = profile.friends_count
ctp.listed_count = profile.listed_count
ctp.favourites_count = profile.favourites_count
ctp.statuses_count = profile.statuses_count
ctp.save()
evolution = EvolutionTwitterProfile(
twitter_profile=ctp,
followers_count=ctp.followers_count,
friends_count=ctp.friends_count,
listed_count=ctp.listed_count,
favourites_count=ctp.favourites_count,
statuses_count=ctp.statuses_count,
)
evolution.save()
self.stdout.write(u'Successfully updated Campaign Twitter Profile: %s \n\n' % ctp.twitter_id)
end_date = datetime.date.today()
end_date = datetime.date.fromordinal(end_date.toordinal() - 1)
statuses = TwitterMessage.objects.filter(
twitter_profile=ctp,
created_time__lte=datetime.datetime.combine(end_date, datetime.time()),
loaded_retweets=False,
deleted=False,
).order_by('-created_time')
for status in [s for s in statuses]:
if status.retweets > 0:
objs = api.retweets(id=status.twitter_id, count=100)
for obj in objs:
try:
twr, created = TwitterRetweet.objects.get_or_create(
message=status,
author_twitter_id=obj.author.id_str,
defaults={
'created_time': obj.created_at,
},
)
twr.save()
self.stdout.write(u'Inserted retweet %s' % obj.id_str)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='TwitterRetweet',
error=u'%s: %s' % (Exception, str(e)),
response=obj.id_str,
)
err.save()
self.stdout.write(u'Inserted error message: %s %s' % (Exception, str(e)))
status.loaded_retweets = True
status.save()
else:
status.loaded_retweets = True
status.save()
def handle(self, *args, **options):
try:
twc = TwitterConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no TwitterConnection')
twc = None
if twc:
# api client
auth = tweepy.OAuthHandler(
settings.TW_CONSUMER_KEY,
settings.TW_CONSUMER_SECRET,
)
auth.set_access_token(twc.access_token, twc.access_token_secret)
api = tweepy.API(auth)
if args:
btps = BrandTwitterProfile.objects.filter(twitter_id__in=args,
first_load=True,
deleted=False)
else:
btps = BrandTwitterProfile.objects.filter(first_load=True,
deleted=False)
error = False
# do first load
for btp in [b for b in btps]:
profile = api.get_user(id=btp.twitter_id)
btp.name = profile.name
btp.screen_name = profile.screen_name
btp.followers_count = profile.followers_count
btp.friends_count = profile.friends_count
btp.listed_count = profile.listed_count
btp.favourites_count = profile.favourites_count
btp.statuses_count = profile.statuses_count
btp.save()
evolution = EvolutionTwitterProfile(
twitter_profile=btp,
followers_count=btp.followers_count,
friends_count=btp.friends_count,
listed_count=btp.listed_count,
favourites_count=btp.favourites_count,
statuses_count=btp.statuses_count,
)
evolution.save()
self.stdout.write(
u'Successfully updated Brand Twitter Profile: %s \n\n' %
btp.twitter_id)
page = 1
main_loop = True
while main_loop:
statuses = api.user_timeline(user_id=btp.twitter_id,
page=page,
count=200)
if statuses:
for status in statuses:
try:
twm = TwitterMessage.objects.get(
twitter_profile=btp,
twitter_id__exact=status.id_str,
deleted=False,
)
except TwitterMessage.DoesNotExist:
try:
twm = TwitterMessage(
twitter_profile=btp,
twitter_id=status.id_str,
author_twitter_id=status.author.id_str,
message=status.text,
created_time=status.created_at,
favorites=status.favorite_count,
retweets=status.retweet_count,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='TwitterMessage',
error=u'%s: %s' % (Exception, str(e)),
response=status.id_str,
)
err.save()
self.stdout.write(
u'Inserted error message: %s %s' %
(Exception, str(e)))
else:
twm.save()
self.stdout.write(u'Inserted message %s' %
twm.twitter_id)
else:
self.stdout.write(
u'Message already exists: %s' %
twm.twitter_id)
main_loop = False
break
page += 1 # next page
self.stdout.write(u'Going to next page: %s\n\n' % page)
else:
main_loop = False
def handle(self, *args, **options):
try:
twc = TwitterConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no TwitterConnection')
twc = None
if twc:
# api client
auth = tweepy.OAuthHandler(
settings.TW_CONSUMER_KEY,
settings.TW_CONSUMER_SECRET,
)
auth.set_access_token(twc.access_token, twc.access_token_secret)
api = tweepy.API(auth)
if args:
atps = AdvertiserTwitterProfile.objects.filter(twitter_id__in=args, first_load=True, deleted=False)
else:
atps = AdvertiserTwitterProfile.objects.filter(first_load=True, deleted=False)
error = False
# do first load
for atp in [a for a in atps]:
profile = api.get_user(id=atp.twitter_id)
atp.name = profile.name
atp.screen_name = profile.screen_name
atp.followers_count = profile.followers_count
atp.friends_count = profile.friends_count
atp.listed_count = profile.listed_count
atp.favourites_count = profile.favourites_count
atp.statuses_count = profile.statuses_count
atp.save()
evolution = EvolutionTwitterProfile(
twitter_profile=atp,
followers_count=atp.followers_count,
friends_count=atp.friends_count,
listed_count=atp.listed_count,
favourites_count=atp.favourites_count,
statuses_count=atp.statuses_count,
)
evolution.save()
self.stdout.write(u'Successfully updated Advertiser Twitter Profile: %s \n\n' % atp.twitter_id)
end_date = datetime.date.today()
end_date = datetime.date.fromordinal(end_date.toordinal() - 1)
statuses = TwitterMessage.objects.filter(
twitter_profile=atp,
created_time__lte=datetime.datetime.combine(end_date, datetime.time()),
loaded_retweets=False,
deleted=False,
).order_by('-created_time')
for status in [s for s in statuses]:
if status.retweets > 0:
objs = api.retweets(id=status.twitter_id, count=100)
for obj in objs:
try:
twr, created = TwitterRetweet.objects.get_or_create(
message=status,
author_twitter_id=obj.author.id_str,
defaults={
'created_time': obj.created_at,
},
)
twr.save()
self.stdout.write(u'Inserted retweet %s' % obj.id_str)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='TwitterRetweet',
error=u'%s: %s' % (Exception, str(e)),
response=obj.id_str,
)
err.save()
self.stdout.write(u'Inserted error message: %s %s' % (Exception, str(e)))
status.loaded_retweets = True
status.save()
else:
status.loaded_retweets = True
status.save()
def handle(self, *args, **options):
# get some access_token
try:
fbc = FacebookConnection.objects.filter(
expires__gt=datetime.datetime.now(), deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no FacebookConnection')
fbc = None
if fbc:
# api client
client = FacebookClient(
client_id=settings.FB_CLIENT_ID,
client_secret=settings.FB_CLIENT_SECRET,
access_token=fbc.access_token,
)
if args:
cfps = CampaignFacebookPage.objects.filter(
facebook_id__in=args, deleted=False)
else:
cfps = CampaignFacebookPage.objects.filter(deleted=False)
error = False
# do daily load
for cfp in [c for c in cfps]:
if error:
break
# page info
try:
cfp_info = client.obj_id(cfp.facebook_id)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
else:
if not cfp_info:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
cfp_info = client.obj_id(cfp.facebook_id)
if not cfp_info:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
cfp.name = cfp_info[u'name']
cfp.link = cfp_info[u'link']
cfp.talking_about = cfp_info[u'talking_about_count']
cfp.likes = cfp_info[u'likes']
cfp.save()
evolution = EvolutionFacebookPageLike(
facebook_page=cfp,
likes=cfp_info[u'likes'],
)
evolution.save()
self.stdout.write(
u'Successfully updated Campaign Facebook Page: %s \n\n' %
cfp.facebook_id)
main_loop = True
until = None
while main_loop:
if error:
break
params = {u'limit': 50}
if until:
params.update({u'until': until})
# load feed
try:
feed = client.feed(cfp.facebook_id, **params)
except FacebookGenericError:
if FacebookGenericError.code == u'17':
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
else:
if not feed:
fbc, access_token = FacebookConnection.renew_client(
fbc)
client._access_token = access_token
feed = client.feed(cfp.facebook_id, **params)
if not feed:
error = True
continue
#raise CommandError(u'\nThe limit of all robots was reached.')
if not u'data' in feed or not len(feed[u'data']):
main_loop = False
cfp.save()
continue
for row in feed[u'data']:
try:
fbm = FacebookMessage.objects.get(
facebook_id__exact=row[u'id'], deleted=False)
except FacebookMessage.DoesNotExist:
try:
if u'likes' in row and u'count' in row[
u'likes']:
likes = row[u'likes'][u'count']
else:
likes = 0
if u'comments' in row and u'count' in row[
u'comments']:
comments = row[u'comments'][u'count']
else:
comments = 0
content = u''
if u'message' in row:
content = u'%s' % row[u'message']
if u'story' in row:
content = u'%s %s' % (content,
row[u'story'])
if u'picture' in row:
content = u'%s %s' % (content,
row[u'picture'])
if u'link' in row:
content = u'%s %s' % (content,
row[u'link'])
fbm = FacebookMessage(
facebook_page=cfp,
facebook_id=row[u'id'],
author_facebook_id=row[u'from'][u'id'],
message=content,
created_time=datetime.datetime.strptime(
row[u'created_time'],
'%Y-%m-%dT%H:%M:%S+0000'),
message_type=row[u'type'],
likes=0,
comments=0,
shares=row[u'shares'][u'count']
if u'shares' in row else 0,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='FacebookMessage',
error=u'%s: %s' % (Exception, str(e)),
response=row,
)
err.save()
self.stdout.write(
u'Inserted error message: %s %s' %
(Exception, str(e)))
else:
fbm.save()
self.stdout.write(u'Inserted message %s' %
fbm.facebook_id)
else:
self.stdout.write(u'Message already exists: %s' %
fbm.facebook_id)
self.stdout.write(u'Stopping...')
main_loop = False
break
if u'paging' in feed and u'next' in feed[u'paging']:
until = urlparse.parse_qs(
urlparse.urlparse(
feed[u'paging'][u'next']).query)[u'until'][0]
self.stdout.write(u'Going to next page \n\n\n')
else:
main_loop = False
cfp.save()
def handle(self, *args, **options):
try:
twc = TwitterConnection.objects.filter(deleted=False)[0]
except IndexError:
self.stdout.write(u'Error: there is no TwitterConnection')
twc = None
if twc:
# api client
auth = tweepy.OAuthHandler(
settings.TW_CONSUMER_KEY,
settings.TW_CONSUMER_SECRET,
)
auth.set_access_token(twc.access_token, twc.access_token_secret)
api = tweepy.API(auth)
if args:
ctps = CampaignTwitterProfile.objects.filter(twitter_id__in=args, first_load=False, deleted=False)
else:
ctps = CampaignTwitterProfile.objects.filter(first_load=False, deleted=False)
error = False
# do first load
for ctp in [c for c in ctps]:
profile = api.get_user(id=ctp.twitter_id)
ctp.name = profile.name
ctp.screen_name = profile.screen_name
ctp.followers_count = profile.followers_count
ctp.friends_count = profile.friends_count
ctp.listed_count = profile.listed_count
ctp.favourites_count = profile.favourites_count
ctp.statuses_count = profile.statuses_count
ctp.save()
evolution = EvolutionTwitterProfile(
twitter_profile=ctp,
followers_count=ctp.followers_count,
friends_count=ctp.friends_count,
listed_count=ctp.listed_count,
favourites_count=ctp.favourites_count,
statuses_count=ctp.statuses_count,
)
evolution.save()
self.stdout.write(u'Successfully updated Campaign Twitter Profile: %s \n\n' % ctp.twitter_id)
try:
page = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='TwitterMessage',
path=u'/%s/timeline' % ctp.twitter_id,
).offset
page = int(page)
except APIPagination.DoesNotExist:
page = 1
main_loop = True
while main_loop:
statuses = api.user_timeline(user_id=ctp.twitter_id, page=page, count=200)
if statuses:
for status in statuses:
try:
twm = TwitterMessage.objects.get(
twitter_profile=ctp,
twitter_id__exact=status.id_str,
deleted=False,
)
except TwitterMessage.DoesNotExist:
try:
twm = TwitterMessage(
twitter_profile=ctp,
twitter_id=status.id_str,
author_twitter_id=status.author.id_str,
message=status.text,
created_time=status.created_at,
favorites=status.favorite_count,
retweets=status.retweet_count,
)
except Exception, e:
err = APIError(
app_name=u'metrics_social',
model_name='TwitterMessage',
error=u'%s: %s' % (Exception, str(e)),
response=status.id_str,
)
err.save()
self.stdout.write(u'Inserted error message: %s %s' % (Exception, str(e)))
else:
twm.save()
self.stdout.write(u'Inserted message %s' % twm.twitter_id)
else:
self.stdout.write(u'Message already exists: %s' % twm.twitter_id)
try:
stopped_at = APIPagination.objects.get(
app_name=u'metrics_social',
model_name='TwitterMessage',
path=u'/%s/timeline' % ctp.twitter_id,
)
stopped_at.offset = page
stopped_at.save()
except APIPagination.DoesNotExist:
stopped_at = APIPagination(
app_name=u'metrics_social',
model_name='TwitterMessage',
path=u'/%s/timeline' % ctp.twitter_id,
offset=page,
)
stopped_at.save()
page += 1 # next page
self.stdout.write(u'Going to next page: %s\n\n' % page)
else:
main_loop = False
ctp.first_load = True
ctp.save()