def save_handle_tweets(self): savedtweets = [] updatedtweets = [] for tweet in self._parsed_tweets: if tweet['add']: del tweet['add'] cursor = dbs.cursor() try: tweet_query, tweet_data = safe_insert( "tracker_handletweet", tweet) cursor.execute(tweet_query, tweet_data) # HandleTweet(**tweet).save() savedtweets.append(tweet['tweet_id']) except MySQLdb.IntegrityError: uniquecolumns = ["tweet_id", "handle_id"] tweet_query, tweet_data = safe_update( "tracker_handletweet", uniquecolumns, tweet) cursor.execute(tweet_query, tweet_data) # HandleTweet.objects.filter(tweet_id=tweet['tweet_id'],handle_id=tweet['handle_id']).update(**tweet) updatedtweets.append(tweet['tweet_id']) except Exception as e: print e print "REQUESTS: %s" % (self.requests) cursor.close() continue dbs.commit() cursor.close() print "%s Tweets Saved" % (len(savedtweets)) print "%s Tweets Updated" % (len(updatedtweets))
def save_hashtag_users(self): savedusers = [] updatedusers = [] for user in self._parsed_users: cursor = dbs.cursor() try: user_query, user_data = safe_insert("tracker_hashtaguser", user) cursor.execute(user_query, user_data) savedusers.append(user['user_id']) except MySQLdb.IntegrityError: uniquecolumns = ["user_id"] user_query, user_data = safe_update("tracker_hashtaguser", uniquecolumns, user) cursor.execute(user_query, user_data) updatedusers.append(user['user_id']) except Exception as e: print e print "REQUESTS: %s" % (self.requests) cursor.close() continue dbs.commit() cursor.close() print "%s Users Saved" % (len(savedusers)) print "%s Users Updated" % (len(updatedusers))
def save_hashtag_tweets(self): savedtweets = [] updatedtweets = [] for tweet in self._parsed_tweets: cursor = dbs.cursor() try: tweet_query, tweet_data = safe_insert("tracker_hashtagtweet", tweet) cursor.execute(tweet_query, tweet_data) savedtweets.append(tweet['tweet_id']) except MySQLdb.IntegrityError: uniquecolumns = ["tweet_id", "hashtag_id"] tweet_query, tweet_data = safe_update("tracker_hashtagtweet", uniquecolumns, tweet) cursor.execute(tweet_query, tweet_data) updatedtweets.append(tweet['tweet_id']) except Exception as e: print e print "REQUESTS: %s" % (self.requests) cursor.close() continue dbs.commit() cursor.close() print "%s Tweets Saved" % (len(savedtweets)) print "%s Tweets Updated" % (len(updatedtweets))
def savePosts(self, handle, postslist): savedposts = [] updatedposts = [] for p in postslist: posts = {} posts['handle_id'] = handle.id posts['likes'] = 0 posts['comments'] = 0 posts['likes'] = p['likes']['count'] posts['comments'] = p['comments']['count'] posts['postid'] = p.get('id').split('_')[0] if p.get('caption'): posts['caption'] = p['caption']['text'] else: posts['caption'] = "" posts['caption'] = posts['caption'].encode('unicode_escape')[:255] posts['tags'] = ",".join( [tag.encode('unicode_escape') for tag in p['tags']]) posts['author'] = p.get('user').get('id') posts['url'] = p['link'] posts['postimg'] = p['images']['thumbnail']['url'] raw_published = p.get('created_time') dt_published = datetime.datetime.fromtimestamp(long(raw_published)) # indiatz_published = dt_published+self._india_offset posts['published'] = dt_published posts['published_date'] = dt_published.strftime('%Y-%m-%d') posts['lastupdated'] = datetime.datetime.now() posts['posttype'] = p.get('type') posts['views'] = 0 posts['active'] = True # pp.pprint(posts) cursor = dbs.cursor() try: post_query, post_data = safe_insert( "tracker_instagramhandlepost", posts) cursor.execute(post_query, post_data) savedposts.append(posts['postid']) except MySQLdb.IntegrityError: del posts['views'] uniquecolumns = ["handle_id", "postid"] post_query, post_data = safe_update( "tracker_instagramhandlepost", uniquecolumns, posts) cursor.execute(post_query, post_data) updatedposts.append(posts['postid']) except Exception as e: print e, "for postid " + posts['postid'] dbs.commit() cursor.close() print "%s Posts Saved" % (len(savedposts)) print "%s Posts Updated" % (len(updatedposts))
def getBotUser(self, handle): print "HANDLE ==> %s %s" % (handle.id, handle.name) handle_query = handle.name igurl = "https://www.instagram.com/%s/?__a=1" igurl = igurl % (handle_query) print 'API URL==>\n%s' % (igurl) response = urlopen(igurl) data = json.loads(response.read()) if data.get('user'): userdata = data['user'] userdetails = {} userdetails['uniqueid'] = userdata['id'] userdetails['name'] = userdata['full_name'].encode( 'unicode_escape') userdetails['description'] = userdata.get( "biography", "")[:500].encode('unicode_escape') userdetails['posts'] = userdata['media']['count'] userdetails['friends'] = userdata['follows']['count'] userdetails['followers'] = userdata['followed_by']['count'] try: userdetails['picture'] = userdata['profile_pic_url'] except KeyError: userdetails['picture'] = "" else: print "User Data Not Found" return False cursor = dbs.cursor() try: user_query, user_data = safe_insert("tracker_instagramuser", userdetails) cursor.execute(user_query, user_data) # FacebookUser(**userdetails).save() print 'Saved Instagram User: %s' % (userdetails['uniqueid']) except MySQLdb.IntegrityError: uniquecolumns = ["uniqueid"] user_query, user_data = safe_update("tracker_instagramuser", uniquecolumns, userdetails) cursor.execute(user_query, user_data) # FacebookUser.objects.filter(uniqueid=userdetails['uniqueid']).update(**userdetails) print 'Updated Instagram User: %s' % (userdetails['uniqueid']) except Exception as e: print e cursor.close() return False dbs.commit() cursor.close() return False
def parse_followers(self, followers): newfollowers = 0 updatedfollowers = 0 followers.reverse() for user_obj in followers: utc_offset = self.checkifnull(user_obj.get('utc_offset', 0)) time_zone = self.checkifnull(user_obj.get('time_zone', 0)) userdetails = { 'handle_id':self.handle.id, 'user_id': user_obj.get('id_str'), 'name': user_obj.get('name').encode('unicode_escape'), 'screen_name': user_obj.get('screen_name').encode('unicode_escape'), 'created_at': self.parse_date_str(user_obj['created_at']).strftime(format), 'tweets': user_obj.get('statuses_count', 0), 'description': user_obj.get('description', '').encode('unicode_escape'), 'followers': user_obj.get('followers_count', 0), 'favorites': user_obj.get('favourites_count', 0), 'listed': user_obj.get('listed_count', 0), 'friends': user_obj.get('friends_count', 0), 'profile_image_url': user_obj.get('profile_image_url', ''), 'utc_offset': utc_offset, 'time_zone': time_zone, 'location': user_obj.get('location', '').encode('unicode_escape'), 'verified': user_obj.get('verified', False), 'lang': user_obj.get('lang', 'en'), 'insertdate': todaydate } cursor = dbs.cursor() try: user_query,user_data = safe_insert("tracker_twitterfollowers",userdetails) cursor.execute(user_query, user_data) newfollowers += 1 # print 'Saved Follower: %s for Date: %s'%(userdetails['user_id'], userdetails['insertdate']) except MySQLdb.IntegrityError: uniquecolumns = ["handle_id","user_id"] user_query,user_data = safe_update("tracker_twitterfollowers",uniquecolumns,userdetails) cursor.execute(user_query, user_data) updatedfollowers += 1 # print 'Updated Follower: %s for Date: %s'%(userdetails['user_id'], userdetails['insertdate']) except Exception as e: print e cursor.close() return False dbs.commit() cursor.close() print "New Followers: %s"%(newfollowers) print "Updated Followers: %s"%(updatedfollowers)
def handle_videos(self): ythandles = Handle.objects.filter(platform_id=3, status=1) handle_no = 1 for handle in ythandles: api_key = self.get_apikey() self.youtube = build('youtube', 'v3', developerKey=api_key) self.handle = handle print "HANDLE ==> %s" % (handle.name) youtubeid = handle.uniqueid chdetails = self.getChannel(id=youtubeid) if chdetails: chdetails['handle_id'] = handle.pk playlistid = chdetails['playlist'] # pp.pprint(chdetails) cursor = dbs.cursor() try: channel_query, channel_data = safe_insert( "tracker_youtubechannel", chdetails) cursor.execute(channel_query, channel_data) # YoutubeChannel(**chdetails).save() print 'Saved Youtube Channel: %s' % ( chdetails['youtubeid']) except MySQLdb.IntegrityError: uniquecolumns = ["youtubeid"] channel_query, channel_data = safe_update( "tracker_youtubechannel", uniquecolumns, chdetails) cursor.execute(channel_query, channel_data) # YoutubeChannel.objects.filter(youtubeid = chdetails['youtubeid']).update(**chdetails) print 'Updated Youtube Channel: %s' % ( chdetails['youtubeid']) except Exception as e: print e cursor.close() continue dbs.commit() cursor.close() self.nextToken = None self.page = 1 self.getPlaylist(playlistid=playlistid, since=self.sincedate) print "HANDLE NO: %s" % (handle_no) handle_no += 1 print "sleep\n" sleep(3)
def getUser(self, handle, data, apiurl='firsturl'): print "HANDLE ==> %s %s" % (handle.id, handle.name) print 'API URL==>\n%s' % (apiurl) userdata = data['data'] if userdata.get("id"): userdetails = {} userdetails['uniqueid'] = userdata['id'] userdetails['name'] = userdata['full_name'].encode( 'unicode_escape') userdetails['description'] = userdata.get( "bio", "")[:500].encode('unicode_escape') userdetails['posts'] = userdata['counts']['media'] userdetails['friends'] = userdata['counts']['follows'] userdetails['followers'] = userdata['counts']['followed_by'] try: userdetails['picture'] = userdata['profile_picture'] except KeyError: userdetails['picture'] = "" # pp.pprint(userdetails) else: print "User Data Not Found" return False cursor = dbs.cursor() try: user_query, user_data = safe_insert("tracker_instagramuser", userdetails) cursor.execute(user_query, user_data) # FacebookUser(**userdetails).save() print 'Saved Instagram User: %s' % (userdetails['uniqueid']) except MySQLdb.IntegrityError: uniquecolumns = ["uniqueid"] user_query, user_data = safe_update("tracker_instagramuser", uniquecolumns, userdetails) cursor.execute(user_query, user_data) # FacebookUser.objects.filter(uniqueid=userdetails['uniqueid']).update(**userdetails) print 'Updated Instagram User: %s' % (userdetails['uniqueid']) except Exception as e: print e cursor.close() return False dbs.commit() cursor.close() return True
def savePost(self, post): cursor = dbs.cursor() try: post_query, post_data = safe_insert("tracker_instagramhandlepost", post) cursor.execute(post_query, post_data) self.savedposts += 1 except MySQLdb.IntegrityError: # '''not updating comments for api enabled handles i.e status=1''' # if self.handle.status == 1: # del post['comments'] uniquecolumns = ["handle_id", "postid"] post_query, post_data = safe_update("tracker_instagramhandlepost", uniquecolumns, post) cursor.execute(post_query, post_data) self.updatedposts += 1 except Exception as e: print e, "for postid " + post['postid'] dbs.commit() cursor.close()
def getUser(self, handle, data, apiurl='firsturl'): print "HANDLE ==> %s %s" % (handle.id, handle.name) print 'API URL==>\n%s' % (apiurl) if data.get("id"): userdetails = {} userdetails['uniqueid'] = data['id'] userdetails['name'] = data['name'].encode('unicode_escape') userdetails['description'] = data.get( "description", "")[:500].encode('unicode_escape') userdetails['likes'] = data['fan_count'] userdetails['talkingabout'] = data['talking_about_count'] try: userdetails['picture'] = data['picture']['data']['url'] except KeyError: userdetails['picture'] = "" userdetails['verified'] = data['is_verified'] # pp.pprint(userdetails) else: print "User Data Not Found" return False cursor = dbs.cursor() try: user_query, user_data = safe_insert("tracker_facebookuser", userdetails) cursor.execute(user_query, user_data) # FacebookUser(**userdetails).save() print 'Saved Facebook User: %s' % (userdetails['uniqueid']) except MySQLdb.IntegrityError: uniquecolumns = ["uniqueid"] user_query, user_data = safe_update("tracker_facebookuser", uniquecolumns, userdetails) cursor.execute(user_query, user_data) # FacebookUser.objects.filter(uniqueid=userdetails['uniqueid']).update(**userdetails) print 'Updated Facebook User: %s' % (userdetails['uniqueid']) except Exception as e: print e cursor.close() return False dbs.commit() cursor.close() return True
def saveVideos(self, videos): savedvideos = [] updatedvideos = [] defaults = {} defaults['comments'] = 0 defaults['likes'] = 0 defaults['views'] = 0 defaults['dislikes'] = 0 defaults['favorites'] = 0 defaults['sharedonfb'] = 0 for video in videos: video['handle_id'] = self.handle.id newvideo = dict(video.items() + defaults.items()) cursor = dbs.cursor() try: video_query, video_data = safe_insert( "tracker_youtubechannelvideo", newvideo) cursor.execute(video_query, video_data) # YoutubeChannelVideo(**newvideo).save() savedvideos.append(newvideo['youtubeid']) except MySQLdb.IntegrityError: uniquecolumns = ["youtubeid"] video_query, video_data = safe_update( "tracker_youtubechannelvideo", uniquecolumns, video) cursor.execute(video_query, video_data) # YoutubeChannelVideo.objects.filter(youtubeid = video['youtubeid']).update(**video) updatedvideos.append(video['youtubeid']) except Exception as e: print e, "for youtubeid: " + video['youtubeid'] cursor.close() continue dbs.commit() cursor.close() print "%s Videos Saved" % (len(savedvideos)) print "%s Videos Updated" % (len(updatedvideos))
def getUser(self, twitterapi): try: print "HANDLE ==> %s %s" % (self.handle.id, self.handle.name) user_obj = twitterapi.users.show(user_id=self.handle.uniqueid) self.requests += 1 except TwitterHTTPError as e: self.raise_apierror(e) print "REQUESTS: %s" % (self.requests) return False # pp.pprint(user_obj) userdetails = { 'user_id': user_obj.get('id_str'), 'name': user_obj.get('name').encode('unicode_escape'), 'screen_name': user_obj.get('screen_name').encode('unicode_escape'), 'created_at': self.parse_date_str( user_obj['created_at']).strftime('%Y-%m-%d %H:%M:%S'), 'statuses_count': user_obj.get('statuses_count', 0), 'description': user_obj.get('description', '').encode('unicode_escape'), 'followers_count': user_obj.get('followers_count', 0), 'favorites_count': user_obj.get('favourites_count', 0), 'listed_count': user_obj.get('listed_count', 0), 'friends_count': user_obj.get('friends_count', 0), 'profile_image_url': user_obj.get('profile_image_url', ''), 'utc_offset': user_obj.get('utc_offset', 0), 'time_zone': user_obj.get('time_zone', 0), 'location': user_obj.get('location', '').encode('unicode_escape'), 'verified': user_obj.get('verified', False), 'lang': user_obj.get('lang', 'en'), 'lastupdated': datetime.datetime.now() } cursor = dbs.cursor() try: user_query, user_data = safe_insert("tracker_twitteruser", userdetails) cursor.execute(user_query, user_data) # TwitterUser(**userdetails).save() print 'Saved Twitter User: %s' % (userdetails['user_id']) except MySQLdb.IntegrityError: uniquecolumns = ["user_id"] user_query, user_data = safe_update("tracker_twitteruser", uniquecolumns, userdetails) cursor.execute(user_query, user_data) # TwitterUser.objects.filter(user_id = userdetails['user_id']).update(**userdetails) print 'Updated Twitter User: %s' % (userdetails['user_id']) except Exception as e: print e print "REQUESTS: %s" % (self.requests) cursor.close() return False dbs.commit() cursor.close() return True
def getUser(self, handle): # userdetails = {} # userdetails['uniqueid'] = handle.uniqueid # headerpath = ".//*[@id='react-root']/section/main/article/header/section/" # try: # for i in range(1,4): # metricobj = self.driver.find_element_by_xpath(headerpath+"ul/li[%s]/span"%(i)) # metric = metricobj.text.split(' ')[-1] # metrictitle = self.driver.find_element_by_xpath(headerpath+"ul/li[%s]/span/span"%(i)).get_attribute("title") # if metrictitle: # metricscore = int(metrictitle.replace(',','')) # else: # metricscore = int(metricobj.text.split(' ')[0].replace(',','')) # if metric == "following": # metric = "friends" # userdetails[str(metric)] = metricscore # userdetails["name"] = self.driver.find_element_by_xpath(headerpath+"div[2]/h1").text # userdetails["description"] = self.driver.find_element_by_xpath(headerpath+"div[2]/span/span").text[:500] # try: # userdetails["picture"] = self.driver.find_element_by_xpath(".//*[@id='react-root']/section/main/article/header/div/div/img").get_attribute("src") # except: # userdetails["picture"] = "" # except WebDriverException, e: # print e # return False handle_query = handle.name igurl = "https://www.instagram.com/%s/?__a=1" igurl = igurl % (handle_query) response = urlopen(igurl) data = json.loads(response.read()) if data.get('user'): userdata = data['user'] userdetails = {} userdetails['uniqueid'] = userdata['id'] userdetails['name'] = userdata['full_name'].encode( 'unicode_escape') userdetails['description'] = userdata.get( "biography", "")[:500].encode('unicode_escape') userdetails['posts'] = userdata['media']['count'] userdetails['friends'] = userdata['follows']['count'] userdetails['followers'] = userdata['followed_by']['count'] try: userdetails['picture'] = userdata['profile_pic_url'] except KeyError: userdetails['picture'] = "" else: print "User Data Not Found" return False cursor = dbs.cursor() try: user_query, user_data = safe_insert("tracker_instagramuser", userdetails) cursor.execute(user_query, user_data) # FacebookUser(**userdetails).save() print 'Saved Instagram User: %s' % (userdetails['uniqueid']) except MySQLdb.IntegrityError: uniquecolumns = ["uniqueid"] user_query, user_data = safe_update("tracker_instagramuser", uniquecolumns, userdetails) cursor.execute(user_query, user_data) # FacebookUser.objects.filter(uniqueid=userdetails['uniqueid']).update(**userdetails) print 'Updated Instagram User: %s' % (userdetails['uniqueid']) except Exception as e: print e cursor.close() return False dbs.commit() cursor.close() return True
def savePosts(self, handle, postslist): savedposts = [] updatedposts = [] for p in postslist: posts = {} posts['handle_id'] = handle.id posts['likes'] = 0 posts['comments'] = 0 posts['likes'] = p['likes']['summary']['total_count'] posts['comments'] = p['comments']['summary']['total_count'] posts['addedtime'] = datetime.datetime.now() posts['postid'] = postid = p.get('object_id', 'None') posts['fbgraph_id'] = p.get('id').split('_')[1] if (postid == 'None'): posts['postid'] = postid = posts['fbgraph_id'] posts['message'] = message = p.get('message', 'None') if (message == 'None'): posts['message'] = message = p.get('story', '') posts['hashtags'] = self.get_hashtags(posts['message']) posts['message'] = posts['message'].encode('unicode_escape')[:255] if 'updated their cover photo' in posts['message']: print 'Ignoring and not saving this post::', posts['message'] continue posts['author'] = p.get('from').get('id') posts['url'] = postlink = "http://www.facebook.com/" + str( posts['author']) + '/posts/' + str(posts['fbgraph_id']) posts['postimg'] = p.get('picture', 0) raw_published = p.get('created_time') dt_published = datetime.datetime( *time.strptime(raw_published, '%Y-%m-%dT%H:%M:%S+0000')[0:6]) indiatz_published = dt_published + self._india_offset posts['published'] = indiatz_published posts['published_date'] = indiatz_published.strftime('%Y-%m-%d') posts['lastupdated'] = datetime.datetime.now() # posts['lastupdated'] = datetime.datetime.now() - datetime.timedelta(hours=8) #New posts are now available for comments n likes updation # shares = p.get('shares',0) shares = p.get('shares', 0) if (shares != 0): shares = shares.get('count') posts['shares'] = shares posts['posttype'] = p.get('type') posts['statustype'] = p.get('status_type') if (posts['author'] == handle.uniqueid): posts['fanpost'] = 0 else: posts['fanpost'] = 1 posts['tagged'] = 0 posts['active'] = True cursor = dbs.cursor() try: post_query, post_data = safe_insert( "tracker_facebookhandlepost", posts) cursor.execute(post_query, post_data) # FacebookHandlePost(**posts).save() savedposts.append(posts['fbgraph_id']) except MySQLdb.IntegrityError: uniquecolumns = ["handle_id", "fbgraph_id"] post_query, post_data = safe_update( "tracker_facebookhandlepost", uniquecolumns, posts) cursor.execute(post_query, post_data) # FacebookHandlePost.objects.filter(handle_id=handle.id,fbgraph_id=posts['fbgraph_id']).update\ # (posttype=posts['posttype'],statustype=posts['statustype'], # likes=posts['likes'],comments=posts['comments'], # shares=posts['shares'],lastupdated=posts['lastupdated']) updatedposts.append(posts['fbgraph_id']) except Exception as e: print e, "for postid " + posts['fbgraph_id'] dbs.commit() cursor.close() # self.getComments(posts['fbgraph_id']) # gc.collect() print "%s Posts Saved" % (len(savedposts)) print "%s Posts Updated" % (len(updatedposts))