def save_hashtag_users(self):
savedusers = []
updatedusers = []
for user in self._parsed_users:
cursor = dbs.cursor()
try:
user_query, user_data = safe_insert("tracker_hashtaguser",
user)
cursor.execute(user_query, user_data)
savedusers.append(user['user_id'])
except MySQLdb.IntegrityError:
uniquecolumns = ["user_id"]
user_query, user_data = safe_update("tracker_hashtaguser",
uniquecolumns, user)
cursor.execute(user_query, user_data)
updatedusers.append(user['user_id'])
except Exception as e:
print e
print "REQUESTS: %s" % (self.requests)
cursor.close()
continue
dbs.commit()
cursor.close()
print "%s Users Saved" % (len(savedusers))
print "%s Users Updated" % (len(updatedusers))
def save_handle_tweets(self):
savedtweets = []
updatedtweets = []
for tweet in self._parsed_tweets:
if tweet['add']:
del tweet['add']
cursor = dbs.cursor()
try:
tweet_query, tweet_data = safe_insert(
"tracker_handletweet", tweet)
cursor.execute(tweet_query, tweet_data)
# HandleTweet(**tweet).save()
savedtweets.append(tweet['tweet_id'])
except MySQLdb.IntegrityError:
uniquecolumns = ["tweet_id", "handle_id"]
tweet_query, tweet_data = safe_update(
"tracker_handletweet", uniquecolumns, tweet)
cursor.execute(tweet_query, tweet_data)
# HandleTweet.objects.filter(tweet_id=tweet['tweet_id'],handle_id=tweet['handle_id']).update(**tweet)
updatedtweets.append(tweet['tweet_id'])
except Exception as e:
print e
print "REQUESTS: %s" % (self.requests)
cursor.close()
continue
dbs.commit()
cursor.close()
print "%s Tweets Saved" % (len(savedtweets))
print "%s Tweets Updated" % (len(updatedtweets))
def save_hashtag_tweets(self):
savedtweets = []
updatedtweets = []
for tweet in self._parsed_tweets:
cursor = dbs.cursor()
try:
tweet_query, tweet_data = safe_insert("tracker_hashtagtweet",
tweet)
cursor.execute(tweet_query, tweet_data)
savedtweets.append(tweet['tweet_id'])
except MySQLdb.IntegrityError:
uniquecolumns = ["tweet_id", "hashtag_id"]
tweet_query, tweet_data = safe_update("tracker_hashtagtweet",
uniquecolumns, tweet)
cursor.execute(tweet_query, tweet_data)
updatedtweets.append(tweet['tweet_id'])
except Exception as e:
print e
print "REQUESTS: %s" % (self.requests)
cursor.close()
continue
dbs.commit()
cursor.close()
print "%s Tweets Saved" % (len(savedtweets))
print "%s Tweets Updated" % (len(updatedtweets))
def savePosts(self, handle, postslist):
savedposts = []
updatedposts = []
for p in postslist:
posts = {}
posts['handle_id'] = handle.id
posts['likes'] = 0
posts['comments'] = 0
posts['likes'] = p['likes']['count']
posts['comments'] = p['comments']['count']
posts['postid'] = p.get('id').split('_')[0]
if p.get('caption'):
posts['caption'] = p['caption']['text']
else:
posts['caption'] = ""
posts['caption'] = posts['caption'].encode('unicode_escape')[:255]
posts['tags'] = ",".join(
[tag.encode('unicode_escape') for tag in p['tags']])
posts['author'] = p.get('user').get('id')
posts['url'] = p['link']
posts['postimg'] = p['images']['thumbnail']['url']
raw_published = p.get('created_time')
dt_published = datetime.datetime.fromtimestamp(long(raw_published))
# indiatz_published = dt_published+self._india_offset
posts['published'] = dt_published
posts['published_date'] = dt_published.strftime('%Y-%m-%d')
posts['lastupdated'] = datetime.datetime.now()
posts['posttype'] = p.get('type')
posts['views'] = 0
posts['active'] = True
# pp.pprint(posts)
cursor = dbs.cursor()
try:
post_query, post_data = safe_insert(
"tracker_instagramhandlepost", posts)
cursor.execute(post_query, post_data)
savedposts.append(posts['postid'])
except MySQLdb.IntegrityError:
del posts['views']
uniquecolumns = ["handle_id", "postid"]
post_query, post_data = safe_update(
"tracker_instagramhandlepost", uniquecolumns, posts)
cursor.execute(post_query, post_data)
updatedposts.append(posts['postid'])
except Exception as e:
print e, "for postid " + posts['postid']
dbs.commit()
cursor.close()
print "%s Posts Saved" % (len(savedposts))
print "%s Posts Updated" % (len(updatedposts))
def getBotUser(self, handle):
print "HANDLE ==> %s %s" % (handle.id, handle.name)
handle_query = handle.name
igurl = "https://www.instagram.com/%s/?__a=1"
igurl = igurl % (handle_query)
print 'API URL==>\n%s' % (igurl)
response = urlopen(igurl)
data = json.loads(response.read())
if data.get('user'):
userdata = data['user']
userdetails = {}
userdetails['uniqueid'] = userdata['id']
userdetails['name'] = userdata['full_name'].encode(
'unicode_escape')
userdetails['description'] = userdata.get(
"biography", "")[:500].encode('unicode_escape')
userdetails['posts'] = userdata['media']['count']
userdetails['friends'] = userdata['follows']['count']
userdetails['followers'] = userdata['followed_by']['count']
try:
userdetails['picture'] = userdata['profile_pic_url']
except KeyError:
userdetails['picture'] = ""
else:
print "User Data Not Found"
return False
cursor = dbs.cursor()
try:
user_query, user_data = safe_insert("tracker_instagramuser",
userdetails)
cursor.execute(user_query, user_data)
# FacebookUser(**userdetails).save()
print 'Saved Instagram User: %s' % (userdetails['uniqueid'])
except MySQLdb.IntegrityError:
uniquecolumns = ["uniqueid"]
user_query, user_data = safe_update("tracker_instagramuser",
uniquecolumns, userdetails)
cursor.execute(user_query, user_data)
# FacebookUser.objects.filter(uniqueid=userdetails['uniqueid']).update(**userdetails)
print 'Updated Instagram User: %s' % (userdetails['uniqueid'])
except Exception as e:
print e
cursor.close()
return False
dbs.commit()
cursor.close()
return False
def parse_followers(self, followers):
newfollowers = 0
updatedfollowers = 0
followers.reverse()
for user_obj in followers:
utc_offset = self.checkifnull(user_obj.get('utc_offset', 0))
time_zone = self.checkifnull(user_obj.get('time_zone', 0))
userdetails = {
'handle_id':self.handle.id,
'user_id': user_obj.get('id_str'),
'name': user_obj.get('name').encode('unicode_escape'),
'screen_name': user_obj.get('screen_name').encode('unicode_escape'),
'created_at': self.parse_date_str(user_obj['created_at']).strftime(format),
'tweets': user_obj.get('statuses_count', 0),
'description': user_obj.get('description', '').encode('unicode_escape'),
'followers': user_obj.get('followers_count', 0),
'favorites': user_obj.get('favourites_count', 0),
'listed': user_obj.get('listed_count', 0),
'friends': user_obj.get('friends_count', 0),
'profile_image_url': user_obj.get('profile_image_url', ''),
'utc_offset': utc_offset,
'time_zone': time_zone,
'location': user_obj.get('location', '').encode('unicode_escape'),
'verified': user_obj.get('verified', False),
'lang': user_obj.get('lang', 'en'),
'insertdate': todaydate
}
cursor = dbs.cursor()
try:
user_query,user_data = safe_insert("tracker_twitterfollowers",userdetails)
cursor.execute(user_query, user_data)
newfollowers += 1
# print 'Saved Follower: %s for Date: %s'%(userdetails['user_id'], userdetails['insertdate'])
except MySQLdb.IntegrityError:
uniquecolumns = ["handle_id","user_id"]
user_query,user_data = safe_update("tracker_twitterfollowers",uniquecolumns,userdetails)
cursor.execute(user_query, user_data)
updatedfollowers += 1
# print 'Updated Follower: %s for Date: %s'%(userdetails['user_id'], userdetails['insertdate'])
except Exception as e:
print e
cursor.close()
return False
dbs.commit()
cursor.close()
print "New Followers: %s"%(newfollowers)
print "Updated Followers: %s"%(updatedfollowers)
def getVideoStats(self, ids):
video_metrics = {
'comments': 'commentCount',
'likes': 'likeCount',
'views': 'viewCount',
'dislikes': 'dislikeCount',
'favorites': 'favoriteCount'
}
params = {'part': 'statistics', 'maxResults': 50}
if ids is not None:
params['id'] = ids
else:
return None
try:
response = self.youtube.videos().list(**params).execute()
# pp.pprint(response['items'])
except apiclient.errors.HttpError as err:
self.raise_apierror(err)
return None
for videoItem in response['items']:
videostats = {'youtubeid': videoItem['id']}
for metrickey, metricvalue in video_metrics.iteritems():
try:
videostats[metrickey] = videoItem['statistics'][
metricvalue]
except KeyError:
videostats[metrickey] = 0
cursor = dbs.cursor()
try:
uniquecolumns = ["youtubeid"]
video_query, video_data = safe_update(
"tracker_youtubechannelvideo", uniquecolumns, videostats)
cursor.execute(video_query, video_data)
# YoutubeChannelVideo.objects.filter(youtubeid=videoItem['id']).update(**videostats)
except Exception as e:
print e
cursor.close()
continue
dbs.commit()
cursor.close()
# sharedonfb = self.getSharedFB(id)
print 'Updated Stats for %s Videos' % (len(response['items']))
def handle_videos(self):
ythandles = Handle.objects.filter(platform_id=3, status=1)
handle_no = 1
for handle in ythandles:
api_key = self.get_apikey()
self.youtube = build('youtube', 'v3', developerKey=api_key)
self.handle = handle
print "HANDLE ==> %s" % (handle.name)
youtubeid = handle.uniqueid
chdetails = self.getChannel(id=youtubeid)
if chdetails:
chdetails['handle_id'] = handle.pk
playlistid = chdetails['playlist']
# pp.pprint(chdetails)
cursor = dbs.cursor()
try:
channel_query, channel_data = safe_insert(
"tracker_youtubechannel", chdetails)
cursor.execute(channel_query, channel_data)
# YoutubeChannel(**chdetails).save()
print 'Saved Youtube Channel: %s' % (
chdetails['youtubeid'])
except MySQLdb.IntegrityError:
uniquecolumns = ["youtubeid"]
channel_query, channel_data = safe_update(
"tracker_youtubechannel", uniquecolumns, chdetails)
cursor.execute(channel_query, channel_data)
# YoutubeChannel.objects.filter(youtubeid = chdetails['youtubeid']).update(**chdetails)
print 'Updated Youtube Channel: %s' % (
chdetails['youtubeid'])
except Exception as e:
print e
cursor.close()
continue
dbs.commit()
cursor.close()
self.nextToken = None
self.page = 1
self.getPlaylist(playlistid=playlistid, since=self.sincedate)
print "HANDLE NO: %s" % (handle_no)
handle_no += 1
print "sleep\n"
sleep(3)
def getUser(self, handle, data, apiurl='firsturl'):
print "HANDLE ==> %s %s" % (handle.id, handle.name)
print 'API URL==>\n%s' % (apiurl)
userdata = data['data']
if userdata.get("id"):
userdetails = {}
userdetails['uniqueid'] = userdata['id']
userdetails['name'] = userdata['full_name'].encode(
'unicode_escape')
userdetails['description'] = userdata.get(
"bio", "")[:500].encode('unicode_escape')
userdetails['posts'] = userdata['counts']['media']
userdetails['friends'] = userdata['counts']['follows']
userdetails['followers'] = userdata['counts']['followed_by']
try:
userdetails['picture'] = userdata['profile_picture']
except KeyError:
userdetails['picture'] = ""
# pp.pprint(userdetails)
else:
print "User Data Not Found"
return False
cursor = dbs.cursor()
try:
user_query, user_data = safe_insert("tracker_instagramuser",
userdetails)
cursor.execute(user_query, user_data)
# FacebookUser(**userdetails).save()
print 'Saved Instagram User: %s' % (userdetails['uniqueid'])
except MySQLdb.IntegrityError:
uniquecolumns = ["uniqueid"]
user_query, user_data = safe_update("tracker_instagramuser",
uniquecolumns, userdetails)
cursor.execute(user_query, user_data)
# FacebookUser.objects.filter(uniqueid=userdetails['uniqueid']).update(**userdetails)
print 'Updated Instagram User: %s' % (userdetails['uniqueid'])
except Exception as e:
print e
cursor.close()
return False
dbs.commit()
cursor.close()
return True
def savePost(self, post):
cursor = dbs.cursor()
try:
post_query, post_data = safe_insert("tracker_instagramhandlepost",
post)
cursor.execute(post_query, post_data)
self.savedposts += 1
except MySQLdb.IntegrityError:
# '''not updating comments for api enabled handles i.e status=1'''
# if self.handle.status == 1:
# del post['comments']
uniquecolumns = ["handle_id", "postid"]
post_query, post_data = safe_update("tracker_instagramhandlepost",
uniquecolumns, post)
cursor.execute(post_query, post_data)
self.updatedposts += 1
except Exception as e:
print e, "for postid " + post['postid']
dbs.commit()
cursor.close()
def getUser(self, handle, data, apiurl='firsturl'):
print "HANDLE ==> %s %s" % (handle.id, handle.name)
print 'API URL==>\n%s' % (apiurl)
if data.get("id"):
userdetails = {}
userdetails['uniqueid'] = data['id']
userdetails['name'] = data['name'].encode('unicode_escape')
userdetails['description'] = data.get(
"description", "")[:500].encode('unicode_escape')
userdetails['likes'] = data['fan_count']
userdetails['talkingabout'] = data['talking_about_count']
try:
userdetails['picture'] = data['picture']['data']['url']
except KeyError:
userdetails['picture'] = ""
userdetails['verified'] = data['is_verified']
# pp.pprint(userdetails)
else:
print "User Data Not Found"
return False
cursor = dbs.cursor()
try:
user_query, user_data = safe_insert("tracker_facebookuser",
userdetails)
cursor.execute(user_query, user_data)
# FacebookUser(**userdetails).save()
print 'Saved Facebook User: %s' % (userdetails['uniqueid'])
except MySQLdb.IntegrityError:
uniquecolumns = ["uniqueid"]
user_query, user_data = safe_update("tracker_facebookuser",
uniquecolumns, userdetails)
cursor.execute(user_query, user_data)
# FacebookUser.objects.filter(uniqueid=userdetails['uniqueid']).update(**userdetails)
print 'Updated Facebook User: %s' % (userdetails['uniqueid'])
except Exception as e:
print e
cursor.close()
return False
dbs.commit()
cursor.close()
return True
def saveVideos(self, videos):
savedvideos = []
updatedvideos = []
defaults = {}
defaults['comments'] = 0
defaults['likes'] = 0
defaults['views'] = 0
defaults['dislikes'] = 0
defaults['favorites'] = 0
defaults['sharedonfb'] = 0
for video in videos:
video['handle_id'] = self.handle.id
newvideo = dict(video.items() + defaults.items())
cursor = dbs.cursor()
try:
video_query, video_data = safe_insert(
"tracker_youtubechannelvideo", newvideo)
cursor.execute(video_query, video_data)
# YoutubeChannelVideo(**newvideo).save()
savedvideos.append(newvideo['youtubeid'])
except MySQLdb.IntegrityError:
uniquecolumns = ["youtubeid"]
video_query, video_data = safe_update(
"tracker_youtubechannelvideo", uniquecolumns, video)
cursor.execute(video_query, video_data)
# YoutubeChannelVideo.objects.filter(youtubeid = video['youtubeid']).update(**video)
updatedvideos.append(video['youtubeid'])
except Exception as e:
print e, "for youtubeid: " + video['youtubeid']
cursor.close()
continue
dbs.commit()
cursor.close()
print "%s Videos Saved" % (len(savedvideos))
print "%s Videos Updated" % (len(updatedvideos))
def getUser(self, twitterapi):
try:
print "HANDLE ==> %s %s" % (self.handle.id, self.handle.name)
user_obj = twitterapi.users.show(user_id=self.handle.uniqueid)
self.requests += 1
except TwitterHTTPError as e:
self.raise_apierror(e)
print "REQUESTS: %s" % (self.requests)
return False
# pp.pprint(user_obj)
userdetails = {
'user_id':
user_obj.get('id_str'),
'name':
user_obj.get('name').encode('unicode_escape'),
'screen_name':
user_obj.get('screen_name').encode('unicode_escape'),
'created_at':
self.parse_date_str(
user_obj['created_at']).strftime('%Y-%m-%d %H:%M:%S'),
'statuses_count':
user_obj.get('statuses_count', 0),
'description':
user_obj.get('description', '').encode('unicode_escape'),
'followers_count':
user_obj.get('followers_count', 0),
'favorites_count':
user_obj.get('favourites_count', 0),
'listed_count':
user_obj.get('listed_count', 0),
'friends_count':
user_obj.get('friends_count', 0),
'profile_image_url':
user_obj.get('profile_image_url', ''),
'utc_offset':
user_obj.get('utc_offset', 0),
'time_zone':
user_obj.get('time_zone', 0),
'location':
user_obj.get('location', '').encode('unicode_escape'),
'verified':
user_obj.get('verified', False),
'lang':
user_obj.get('lang', 'en'),
'lastupdated':
datetime.datetime.now()
}
cursor = dbs.cursor()
try:
user_query, user_data = safe_insert("tracker_twitteruser",
userdetails)
cursor.execute(user_query, user_data)
# TwitterUser(**userdetails).save()
print 'Saved Twitter User: %s' % (userdetails['user_id'])
except MySQLdb.IntegrityError:
uniquecolumns = ["user_id"]
user_query, user_data = safe_update("tracker_twitteruser",
uniquecolumns, userdetails)
cursor.execute(user_query, user_data)
# TwitterUser.objects.filter(user_id = userdetails['user_id']).update(**userdetails)
print 'Updated Twitter User: %s' % (userdetails['user_id'])
except Exception as e:
print e
print "REQUESTS: %s" % (self.requests)
cursor.close()
return False
dbs.commit()
cursor.close()
return True
def getUser(self, handle):
# userdetails = {}
# userdetails['uniqueid'] = handle.uniqueid
# headerpath = ".//*[@id='react-root']/section/main/article/header/section/"
# try:
# for i in range(1,4):
# metricobj = self.driver.find_element_by_xpath(headerpath+"ul/li[%s]/span"%(i))
# metric = metricobj.text.split(' ')[-1]
# metrictitle = self.driver.find_element_by_xpath(headerpath+"ul/li[%s]/span/span"%(i)).get_attribute("title")
# if metrictitle:
# metricscore = int(metrictitle.replace(',',''))
# else:
# metricscore = int(metricobj.text.split(' ')[0].replace(',',''))
# if metric == "following":
# metric = "friends"
# userdetails[str(metric)] = metricscore
# userdetails["name"] = self.driver.find_element_by_xpath(headerpath+"div[2]/h1").text
# userdetails["description"] = self.driver.find_element_by_xpath(headerpath+"div[2]/span/span").text[:500]
# try:
# userdetails["picture"] = self.driver.find_element_by_xpath(".//*[@id='react-root']/section/main/article/header/div/div/img").get_attribute("src")
# except:
# userdetails["picture"] = ""
# except WebDriverException, e:
# print e
# return False
handle_query = handle.name
igurl = "https://www.instagram.com/%s/?__a=1"
igurl = igurl % (handle_query)
response = urlopen(igurl)
data = json.loads(response.read())
if data.get('user'):
userdata = data['user']
userdetails = {}
userdetails['uniqueid'] = userdata['id']
userdetails['name'] = userdata['full_name'].encode(
'unicode_escape')
userdetails['description'] = userdata.get(
"biography", "")[:500].encode('unicode_escape')
userdetails['posts'] = userdata['media']['count']
userdetails['friends'] = userdata['follows']['count']
userdetails['followers'] = userdata['followed_by']['count']
try:
userdetails['picture'] = userdata['profile_pic_url']
except KeyError:
userdetails['picture'] = ""
else:
print "User Data Not Found"
return False
cursor = dbs.cursor()
try:
user_query, user_data = safe_insert("tracker_instagramuser",
userdetails)
cursor.execute(user_query, user_data)
# FacebookUser(**userdetails).save()
print 'Saved Instagram User: %s' % (userdetails['uniqueid'])
except MySQLdb.IntegrityError:
uniquecolumns = ["uniqueid"]
user_query, user_data = safe_update("tracker_instagramuser",
uniquecolumns, userdetails)
cursor.execute(user_query, user_data)
# FacebookUser.objects.filter(uniqueid=userdetails['uniqueid']).update(**userdetails)
print 'Updated Instagram User: %s' % (userdetails['uniqueid'])
except Exception as e:
print e
cursor.close()
return False
dbs.commit()
cursor.close()
return True
def savePosts(self, handle, postslist):
savedposts = []
updatedposts = []
for p in postslist:
posts = {}
posts['handle_id'] = handle.id
posts['likes'] = 0
posts['comments'] = 0
posts['likes'] = p['likes']['summary']['total_count']
posts['comments'] = p['comments']['summary']['total_count']
posts['addedtime'] = datetime.datetime.now()
posts['postid'] = postid = p.get('object_id', 'None')
posts['fbgraph_id'] = p.get('id').split('_')[1]
if (postid == 'None'):
posts['postid'] = postid = posts['fbgraph_id']
posts['message'] = message = p.get('message', 'None')
if (message == 'None'):
posts['message'] = message = p.get('story', '')
posts['hashtags'] = self.get_hashtags(posts['message'])
posts['message'] = posts['message'].encode('unicode_escape')[:255]
if 'updated their cover photo' in posts['message']:
print 'Ignoring and not saving this post::', posts['message']
continue
posts['author'] = p.get('from').get('id')
posts['url'] = postlink = "http://www.facebook.com/" + str(
posts['author']) + '/posts/' + str(posts['fbgraph_id'])
posts['postimg'] = p.get('picture', 0)
raw_published = p.get('created_time')
dt_published = datetime.datetime(
*time.strptime(raw_published, '%Y-%m-%dT%H:%M:%S+0000')[0:6])
indiatz_published = dt_published + self._india_offset
posts['published'] = indiatz_published
posts['published_date'] = indiatz_published.strftime('%Y-%m-%d')
posts['lastupdated'] = datetime.datetime.now()
# posts['lastupdated'] = datetime.datetime.now() - datetime.timedelta(hours=8) #New posts are now available for comments n likes updation
# shares = p.get('shares',0)
shares = p.get('shares', 0)
if (shares != 0):
shares = shares.get('count')
posts['shares'] = shares
posts['posttype'] = p.get('type')
posts['statustype'] = p.get('status_type')
if (posts['author'] == handle.uniqueid):
posts['fanpost'] = 0
else:
posts['fanpost'] = 1
posts['tagged'] = 0
posts['active'] = True
cursor = dbs.cursor()
try:
post_query, post_data = safe_insert(
"tracker_facebookhandlepost", posts)
cursor.execute(post_query, post_data)
# FacebookHandlePost(**posts).save()
savedposts.append(posts['fbgraph_id'])
except MySQLdb.IntegrityError:
uniquecolumns = ["handle_id", "fbgraph_id"]
post_query, post_data = safe_update(
"tracker_facebookhandlepost", uniquecolumns, posts)
cursor.execute(post_query, post_data)
# FacebookHandlePost.objects.filter(handle_id=handle.id,fbgraph_id=posts['fbgraph_id']).update\
# (posttype=posts['posttype'],statustype=posts['statustype'],
# likes=posts['likes'],comments=posts['comments'],
# shares=posts['shares'],lastupdated=posts['lastupdated'])
updatedposts.append(posts['fbgraph_id'])
except Exception as e:
print e, "for postid " + posts['fbgraph_id']
dbs.commit()
cursor.close()
# self.getComments(posts['fbgraph_id'])
# gc.collect()
print "%s Posts Saved" % (len(savedposts))
print "%s Posts Updated" % (len(updatedposts))