def run(self): followedCount = 0 for accountName in self.template['screen_names']: #Check if followers have been cached followersCache = Utils.cache(accountName + '_scraped') if (followersCache != '1'): #Retrieve followers followers = {} self.browser.get('https://twitter.com/' + accountName + '/followers') #Grab all followers while True: #Wait for page load and get links WebDriverWait(self.browser, 5).until( EC.presence_of_element_located(( By.XPATH, "/html/body/div[1]/div[1]/div[1]/div[2]/main/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/section/div[1]/div[1]" ))) followerLinks = self.browser.find_elements_by_xpath( '/html/body/div[1]/div[1]/div[1]/div[2]/main/div[1]/div[1]/div[1]/div[1]/div[1]/div[2]/section/div[1]/div[1]/div' ) for followerLink in followerLinks: try: #Init db connection DB.execute( '''INSERT OR IGNORE INTO followers(user_name, user_link, followed, followed_at, parent_account) VALUES(?, ?, ?, ?, ?);''', ( followerLink.find_element_by_css_selector( 'a').get_attribute('href').replace( 'https://twitter.com/', ''), followerLink.find_element_by_css_selector( 'a').get_attribute('href'), True if ('Following' in followerLink.text or 'Pending' in followerLink.text) else False, '', accountName, )) except: print('Ignoring...') #Some load without data #Handle infinite scroll (https://dev.to/mr_h/python-selenium-infinite-scrolling-3o12) last_height = self.browser.execute_script( "return document.body.scrollHeight" ) # Get scroll height self.browser.execute_script( "window.scrollTo(0, document.body.scrollHeight);" ) #Scroll to bottom time.sleep(5) # Wait to load page # Calculate new scroll height and compare with last scroll height new_height = self.browser.execute_script( "return document.body.scrollHeight") if new_height == last_height: # If heights are the same it will exit the function break last_height = new_height Utils.cache(accountName + '_scraped', 1, 43200) #Get updated list from db followers = DB.selectAll( "SELECT * FROM followers WHERE parent_account = ? and followed = 0 limit ?;", (accountName, self.template['amount'])) #Loop through followers and follow for follower in followers: #Load user page print("Following " + follower['user_name']) self.browser.get(follower['user_link']) WebDriverWait(self.browser, 10).until( EC.element_to_be_clickable( (By.XPATH, '//span[text()="@' + follower['user_name'] + '"]'))) #Follow user try: followBtn = self.browser.find_element_by_xpath( '//span[text()="Follow"]') followBtn.click() except: print('User already followed') #Update user as followed DB.execute( '''UPDATE followers SET followed = ?, followed_at = ? WHERE id = ?;''', (1, str(datetime.now()), follower['id'])) followedCount += 1 print("Taking a short rest...") time.sleep( randrange(self.template['sleep_delay'] * 0.6, self.template['sleep_delay'])) #Final output print("Followed " + str(followedCount) + " people")
import sys from src.db import DB rows = DB.selectAll( '''SELECT * FROM followers WHERE parent_account = ? AND followed = 0;''', ('VisitLancashire', )) print(rows)
def run(self): unfollowedCount = 0 #Get recently followed recentlyFollowed = DB.selectAll( "select * from `followers` where date(`followed_at`) < date('now', '-5 days') and `followed_back` is NULL;", ()) #Check if followers have followed back for followedUser in recentlyFollowed: self.browser.get(followedUser['user_link']) #Get following back status try: WebDriverWait(self.browser, 5).until( EC.element_to_be_clickable( (By.XPATH, "//span[text()='Following' or text()='Pending']"))) followsBack = self.browser.find_elements_by_xpath( '//span[text()="Follows you"]') if (len(followsBack) > 0): DB.execute( '''UPDATE followers SET followed_back = ? WHERE id = ?;''', (1, followedUser['id'])) else: #Click unfollow self.browser.find_element_by_xpath( '//span[text()="Following" or text()="Pending"]' ).click() #Click confirm WebDriverWait(self.browser, 7).until( EC.element_to_be_clickable( (By.XPATH, "//span[text()='Discard' or text()='Unfollow']"))) self.browser.find_element_by_xpath( '//span[text()="Discard" or text()="Unfollow"]').click( ) #Mark as not followed back DB.execute( '''UPDATE followers SET followed_back = ? WHERE id = ?;''', (0, followedUser['id'])) unfollowedCount = unfollowedCount + 1 print('Stopped following ' + followedUser['user_name']) except Exception as e: print(e) #Mark as unfollowed # DB.execute('''UPDATE followers SET # followed_back = 0 # WHERE id = ?;''', (followedUser['id']) # ) print('Error occured checking follow status of ' + followedUser['user_name']) print("Taking a short rest...") time.sleep( randrange(self.template['sleep_delay'] * 0.6, self.template['sleep_delay'])) #Final output print("Unfollowed " + str(unfollowedCount) + " people")