def main(): try: start = sys.argv[1] except: print('ERROR: Requires URL as the first argument.') quit(0) # Constants ALLDROPDOWN = '//*[@id="selectReadType"]/option[2]' ACTUALIMAGES = '//*[@id="divImage"]//img' IMGGROUPS = '.listing a' TITLE = '.bigChar' NEXT = '//*[(@id = "btnNext")]//src' s = Session( webdriver_path='C:\\Webdrivers\\chromedriver', browser='chrome' ) # ,webdriver_options={'arguments': ['headless', 'disable-gpu']} s.driver.get(start) s.driver.ensure_element_by_css_selector(TITLE) title = s.driver.find_element_by_css_selector(TITLE).text groups = s.driver.find_elements_by_css_selector(IMGGROUPS) s.transfer_driver_cookies_to_session() begin = to_attribute_list(groups, 'href').pop() response = s.get(begin).xpath(ACTUALIMAGES) print(response) s.close() quit(2)
class ChromeTestCase(unittest.TestCase): def setUp(self): self.s = Session( 'chromedriver', browser='chrome', default_timeout=15, webdriver_options={'arguments': ['headless', 'disable-gpu']}) def test_cookie_transfer_to_requests(self): """Tested on http://testing-ground.scraping.pro/login""" self.s.driver.get('http://testing-ground.scraping.pro/login') self.s.driver.find_element_by_id('usr').send_keys('admin') self.s.driver.ensure_element_by_id('pwd').send_keys( '12345', Keys.ENTER) self.s.driver.ensure_element_by_xpath( '//div[@id="case_login"]/h3[@class="success"]') self.s.transfer_driver_cookies_to_session() response = self.s.get( 'http://testing-ground.scraping.pro/login?mode=welcome') success_message = response.xpath( '//div[@id="case_login"]/h3[@class="success"]/text()' ).extract_first() self.assertEqual( success_message, 'WELCOME :)', 'Failed to transfer cookies from Selenium to Requests') def test_cookie_transfer_to_selenium(self): self.s.get('http://testing-ground.scraping.pro/login') self.s.cookies.set('tdsess', 'TEST_DRIVE_SESSION', domain='testing-ground.scraping.pro') self.s.transfer_session_cookies_to_driver() self.s.driver.get( 'http://testing-ground.scraping.pro/login?mode=welcome') success_message = self.s.driver.xpath( '//div[@id="case_login"]/h3[@class="success"]/text()' ).extract_first() self.assertEqual( success_message, 'WELCOME :)', 'Failed to transfer cookies from Requests to Selenium') def tearDown(self): self.s.driver.close()
def main(url): session = Session( webdriver_path='../Chrome Canary/chromedriver.exe', browser='chrome', default_timeout=6, webdriver_options={'arguments': ['disable-logging', 'headless']}) session.driver.get(url) div_content = WebDriverWait(session.driver, 5).until( EC.presence_of_element_located((By.XPATH, "//div[@id='content']"))) print('######## FROM SELENIUM ########') print(div_content.text) print('######## COPYING SESSION FROM SELENIUM TO REQUESTS ########') session.transfer_driver_cookies_to_session() final_response = session.get(url, headers={'user-agent': 'custom requestium'}) soup = BeautifulSoup(final_response.text, 'html.parser') print('######## FROM REQUESTS ########') body_text = soup.find(id="content") print(body_text.text)
class ChromeTestCase(unittest.TestCase): def setUp(self): self.s = Session('chromedriver', browser='chrome', default_timeout=15, webdriver_options={'arguments': ['headless', 'disable-gpu']}) def test_cookie_transfer_to_requests(self): """Tested on http://testing-ground.scraping.pro/login""" self.s.driver.get('http://testing-ground.scraping.pro/login') self.s.driver.find_element_by_id('usr').send_keys('admin') self.s.driver.ensure_element_by_id('pwd').send_keys('12345', Keys.ENTER) self.s.driver.ensure_element_by_xpath('//div[@id="case_login"]/h3[@class="success"]') self.s.transfer_driver_cookies_to_session() response = self.s.get('http://testing-ground.scraping.pro/login?mode=welcome') success_message = response.xpath( '//div[@id="case_login"]/h3[@class="success"]/text()').extract_first() self.assertEqual( success_message, 'WELCOME :)', 'Failed to transfer cookies from Selenium to Requests') def test_cookie_transfer_to_selenium(self): self.s.get('http://testing-ground.scraping.pro/login') self.s.cookies.set('tdsess', 'TEST_DRIVE_SESSION', domain='testing-ground.scraping.pro') self.s.transfer_session_cookies_to_driver() self.s.driver.get('http://testing-ground.scraping.pro/login?mode=welcome') success_message = self.s.driver.xpath( '//div[@id="case_login"]/h3[@class="success"]/text()').extract_first() self.assertEqual( success_message, 'WELCOME :)', 'Failed to transfer cookies from Requests to Selenium') def tearDown(self): self.s.driver.close()
def login_Getcookie(myaccount, mypassword): #webdriver_option来确定是否使用浏览器显示或者静默登陆 rq = Session( webdriver_path= 'C:\Program Files (x86)\Google\Chrome\Application\chromedriver', browser='chrome', default_timeout=15, #webdriver_options={'arguments': ['headless']} ) rq.driver.get("https://passport.weibo.cn/signin/login") inputname = rq.driver.find_element("xpath", '//*[@id="loginName"]') password = rq.driver.find_element_by_xpath('//*[@id="loginPassword"]') login_button = rq.driver.find_element_by_xpath('//*[@id="loginAction"]') rq.driver.implicitly_wait(10) inputname.send_keys(myaccount) password.send_keys(mypassword) login_button.click() rq.driver.implicitly_wait(15) # ver_button = rq.driver.find_element_by_xpath('//*[@id="embed-captcha"]/div/div[2]/div[1]/div[3]') # ver_button.click() #验证码不知道啥时候会有,反正登陆几次把 rq.transfer_driver_cookies_to_session() return rq
class AuM(object): def __init__(self): # Create a session and authenticate self._s = Session( webdriver_path='/usr/lib/chromium-browser/chromedriver', browser='chrome', webdriver_options={"arguments": ["--headless"]}) self._s.headers.update({ 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:63.0) Gecko/20100101 Firefox/63.0' }) self._s.get('https://www.adopteunmec.com') # Maybe not needed # Register a new account rand_s = ''.join( random.choice(string.ascii_lowercase + string.digits) for _ in range(6)) print('email used: francois_%[email protected]' % rand_s) r = self._s.post('https://www.adopteunmec.com/register/index', data={ 'sex': '1', 'day': '03', 'month': '4', 'year': '1997', 'email': '*****@*****.**' % rand_s, 'password': '******', 'password_check': 'Adottami1', 'country': 'fr', 'zipcode': '06000', 'city': 'Nice', 'confirm_city': '0', 'pseudo': 'RedoAA', 'cgu': '1', 'reg_submit': '', 'by_popup': '1', 'PreventChromeAutocomplete': '' }, headers={ "X-Requested-With": "XMLHttpRequest", "Origin": "https://www.adopteunmec.com/", "Referer": "https://www.adopteunmec.com/" }) status = r.json() # If registration were successful, go to redirect page to confirm account if (status['success'] == 1): self._s.get(status['redirect']) else: print('Something went wrong....') self._common_names = ( 'loic', 'marc', 'anthony', 'tom', 'jordan', 'florian', 'jean', 'manu', 'seb', 'alex', 'lilian', 'angelo', 'fred', 'valent', 'fabrice', 'fabien', 'nico', 'thomas', 'sylvain', 'tim', 'karim', 'robin', 'pierre', 'arnaud', 'max', 'luc', 'mike', 'yann', 'oliv', 'yvan', 'jerem', 'michel', 'mat', 'kev', 'damien', 'vinc', 'eric', 'gilles', 'jona', 'bruno', 'simon', 'adri', 'serge', 'tony', 'jul', 'quentin', 'leo', 'step', 'gab', 'david', 'paul', 'killian', 'alvaro', 'ronan', 'anto', 'jb', 'jp', 'jon', 'patrick', 'virgile', 'juju', 'stef', 'franck', 'alan', 'alain', 'albin', 'alban', 'fran', 'cyril', 'laure', 'phil', 'jacques', 'jack', 'ludo', 'chris', 'vic', 'jo', 'charles', 'geoffrey', 'igor', 'ciro', 'erwan', 'fabio', 'guillaume', 'thibaut', 'romain', 'rafa', 'lionel', 'cedric', 'xavier') def _common_name(self, name): return len( filter(lambda x: x is False, map(lambda n: name.lower().find(n) < 0, self._common_names))) > 0 def search_by_region(self, age_min=20, age_max=30, region=1, sex=0): return self.search({ 'age[min]': age_min, 'age[max]': age_max, 'by': 'region', 'region': region, "sex": sex }) def search_by_disance(self, age_min=20, age_max=30, distance=40, sex=0): return self.search({ 'age[min]': age_min, 'age[max]': age_max, 'by': 'distance', 'distance[max]': distance, "sex": sex }) def search(self, criteria=None): if criteria is None: return [] # Go to search page self._s.get('https://www.adopteunmec.com/mySearch') # POST a request r = self._s.post('https://www.adopteunmec.com/mySearch/save', data=criteria) time.sleep(3) # Wait a bit... # Trasnfer cookie to selenium, refresh the page, scroll to end 10 times, and get profiles self._s.transfer_session_cookies_to_driver() self._s.driver.get('https://www.adopteunmec.com/mySearch/results') for i in range(10): self._s.driver.find_element_by_tag_name('html').send_keys(Keys.END) time.sleep(.1) html = BeautifulSoup( self._s.driver.execute_script("return document.body.innerHTML"), 'lxml') self._s.transfer_driver_cookies_to_session() self._s.driver.close() # Might be done before ? # Look for <div> tags containing user info blocks = html.find_all('div', {'class': 'user-infos'}) # Get all <a> tags in a same list all_a = [a for sl in [b.find_all('a') for b in blocks] for a in sl] # Extract profiles ID doing common name checks to avoid visit too may profiles later profiles = [ l.get('href').split('/')[-1] for l in all_a if isinstance(l.get('href'), str) and l.get('href').find('profile') > 0 and len(l.get_text()) > 2 and not self._common_name(l.get_text()) ] return profiles def update_db(self, profiles=[], max_p=None, filename='data/justemenemoi.json'): db = {} try: with open(filename, 'r') as in_f: db = json.load(in_f) except: pass visited = 0 for uid in profiles: # Check if profile already in db if uid not in db: if max_p is not None and visited >= max_p: break visited += 1 url = "https://www.adopteunmec.com/profile/" + uid page = self._s.get(url) html = BeautifulSoup( page.content.decode('utf-8', 'xmlcharrefreplace')) name = html.find('div', {'class': 'username'}).get_text() desc = html.find(text='Description').find_parent('div').find( 'p').get_text() shop = html.find(text='Shopping List').find_parent('div').find( 'p').get_text() # Profile Filtering if desc.find("non renseign") >= 0 or shop.find( "non renseign") >= 0 or len(desc) < 20 or len( shop) < 20: continue img_url = html.find(id='img-current-pic')['src'] img_name = img_url.split('/')[-1] db[uid] = { "profile": url, "name": name, "img": img_name, "age": html.find('span', { 'class': 'age' }).get_text(), "city": html.find('span', { 'class': 'city' }).get_text(), "desc": desc, "shop": shop } # Download and save profile pic pic = self._s.get(img_url, stream=True) pic.raw.decode_content = True with open("data/pics/" + img_name, 'wb') as f: shutil.copyfileobj(pic.raw, f) time.sleep(.5) # Bit of rest... # Write back json json_s = json.dumps( db) # Dump as a string, to write to file and as JS var with open(filename, 'w') as out_f: out_f.write(json_s) with open(filename + '.js', 'w') as out_f: out_f.write("data = ") out_f.write(json_s)
class Downloader(): def __init__(self, username, password, driver_path=None, download_path=None, headless=True, logger=None): if not logger: logging.basicConfig(level=logging.DEBUG) self.logger = logging.getLogger(__name__) self.logger.setLevel('DEBUG') else: self.logger = logger self._username = username self._password = password self.driver_path = driver_path self.download_path = download_path self.logger = logging.getLogger('odigo_downloader.downloader') self.url = 'https://enregistreur.prosodie.com/odigo4isRecorder/EntryPoint?serviceName=LoginHandler' self.headless = headless self.validated = False self.active = False def __str__(self): return f"\nDOWNLOAD PATH: {self.download_path}\nOPTIONS: {self.webdriver_options}\n" \ f"DRIVER PATH: {self.driver_path}\nUSERNAME: {self._username}\nURL: {self.url}" def setup_selenium_browser(self): if self.active: return f"Session/Browser already active. Cannot have two concurrent sessions/browsers" options = webdriver.ChromeOptions() prefs = { 'download.default_directory': self.download_path, 'download.prompt_for_download': False, 'download.directory_upgrade': True, 'safebrowsing.enabled': False, 'safebrowsing.disable_download_protection': True } options.add_experimental_option('prefs', prefs) if self.headless: options.add_argument('--headless') self.browser = webdriver.Chrome(self.driver_path, options=options) if self.headless: self.browser.command_executor._commands["send_command"] = ( "POST", '/session/$sessionId/chromium/send_command') params = { 'cmd': 'Page.setDownloadBehavior', 'params': { 'behavior': 'allow', 'downloadPath': self.download_path } } command_result = self.browser.execute("send_command", params) for key in command_result: self.logger.debug("result:" + key + ":" + str(command_result[key])) self.active = True def setup_requestium_session(self): if self.active: return f"Session/Browser already active. Cannot have two concurrent sessions/browsers" if self.headless: webdriver_options = {'arguments': ['headless']} else: webdriver_options = {} self.logger.debug( f"Creating Session object with values: {webdriver_options}") self.session = Session(webdriver_path=self.driver_path, browser='chrome', default_timeout=15, webdriver_options=webdriver_options) self.active = True def login_requestium(self): if self.active: raise CustomException(f"Cannot have two active sessions/browsers") self.setup_requestium_session() self.logger.debug(f"Going to URL: {self.url}") self.session.driver.get(self.url) self.logger.debug(f"Entering credentials") self.session.driver.ensure_element_by_name('mail').send_keys( self._username) self.session.driver.ensure_element_by_name('password').send_keys( self._password) self.session.driver.ensure_element_by_name('valider').click() self.validated = True def login_selenium(self): if self.active: raise CustomException(f"Cannot have two active sessions/browsers") self.setup_selenium_browser() self.browser.get(self.url) self.browser.find_element_by_name('mail').send_keys(username) self.browser.find_element_by_name('password').send_keys(password) self.browser.find_element_by_name('valider').click() return def download_mp3(self, path=None, ref=None, xpath=None): self.logger.info( f"\ndownload_mp3 called with:\nPATH: {path},\nREF: {ref},\nXPATH: {xpath}" ) if ref is not None and xpath is None: self.session.driver.ensure_element_by_class_name( 'x-action-col-icon').click() elif xpath is not None and ref is None: self.session.driver.ensure_element_by_xpath(xpath).click() else: self.logger.error("Cannot use both reference number and xpath") return self.session.driver.switch_to.frame('result_frame') time.sleep(1) # Get URL of mp3 file src = self.session.driver.ensure_element_by_id( 'messagePlayer').get_attribute('src') # Selenium --> Requests self.session.transfer_driver_cookies_to_session() # Download r = self.session.get(src, stream=True) if path is None: if ref is None: # Get ref number soap = BeautifulSoup(self.session.driver.page_source, 'lxml') ref = soap.findAll('div', class_='x-grid-cell-inner')[1].text path = '%s.mp3' % ref if r.status_code == 200: with open(path, 'wb') as f: for chunk in r.iter_content(1024 * 2014): f.write(chunk) else: return 1 # Requests --> Selenium self.session.transfer_session_cookies_to_driver() self.session.driver.switch_to.default_content() return def download_mp3_by_ref(self, ref, path=None): self.login_requestium() self.search_by_ref(ref) result = self.download_mp3(path, ref) if result == 1: return 1 self.session.driver.close() def download_mp3_by_csv(self, csv_path, download_dir=None): if download_dir is None: download_dir = self.download_path self.login_requestium() refs = pd.read_csv(csv_path, sep=';').Name length = len(refs) for i, ref in enumerate(refs): sys.stdout.write('\r') sys.stdout.write('downloading: %s/%s' % (i + 1, length)) sys.stdout.flush() self.search_by_ref(ref) mp3_path = None if download_dir is not None: file_name = '%s.mp3' % ref mp3_path = os.path.join(download_dir, file_name) result = self.download_mp3(path=mp3_path, ref=ref) if result == 1: return 1 sys.stdout.write('\n') sys.stdout.flush() self.session.driver.close() return "Finished" def search_by_ref(self, ref): self.session.driver.get(self.url) self.session.driver.ensure_element_by_name('refEr').send_keys(ref) self.session.driver.ensure_element_by_id('button-1009').click() def change_date_format(self, date): try: correct_string = date.strptime(str(date.date()), '%Y-%m-%d').strftime('%d-%m-%Y') return correct_string except Exception as e: raise e def change_time_format(self, date): try: correct_string = date.strptime( str(date.hour) + ':' + str(date.minute), "%H:%M").strftime("%I:%M %p") if correct_string[0] == "0": return correct_string[1::] else: return correct_string except Exception as e: raise e def ceil_dt(self, dt, delta): """Round up to the nearest half hour""" return dt + (datetime.datetime.min - dt) % delta def set_range(self, now): """ Takes current datetime and finds the nearest, previous half hour. Returns the appropriate start and end times and date """ # Format: '10-19-2018' # Format: '12:00 AM' hour_ago = now - datetime.timedelta(minutes=60) rounded = self.ceil_dt(hour_ago, datetime.timedelta(minutes=30)) start_date = self.change_date_format(rounded) start_time = self.change_time_format(rounded) thirty_mins = datetime.timedelta(minutes=30) end_date = start_date end_time = self.change_time_format(rounded + thirty_mins) return (start_date, start_time, end_date, end_time) def search_by_range(self, start_date, start_time, end_date, end_time): """ Doesn't work correctly. Date seems to work but time not so much. Search records on www.prosodie.com by date range and return session. Input: s -- Requestium session (required | type: requestium.requestium.Session); start_date -- start date (not required | type: str). Format: 'mm:dd:yyyy'. Example: '03-05-1991'; start_time -- start time (not required | type: str). Example: '12:00 AM'; end_date -- end date (not required | type: str). Format: 'mm:dd:yyyy'. Example: '03-05-1991'; end_time -- end time (not required | type: str). Example: '12:00 PM'. Output: s -- Requestium session (type: requestium.requestium.Session). """ if start_date: self.browser.find_element_by_name('dateDebut').send_keys( start_date) if start_time: self.browser.find_element_by_name('heureDebut').send_keys( start_time) if end_date: self.browser.find_element_by_name('dateFin').send_keys(end_date) if end_time: self.browser.find_element_by_name('heureFin').send_keys(end_time) self.browser.find_element_by_id('button-1009').click() return def download_all_half_hour(self): self.logger.debug(f"Downloading calls from last half hour") self.logger.debug(f"Login check...") if not self.validated: self.logger.debug(f"Not logged in. Validating") self.login_selenium() self.logger.debug(f"Logged in.") self.logger.debug(f"Getting search range") search_range = self.set_range(datetime.datetime.now()) sleep(2) self.logger.debug(f"Applying filters") self.browser.find_element_by_id("criteres-inputEl").send_keys('_EN') self.search_by_range(*search_range) sleep(5) self.logger.debug(f"Downloading results to {self.download_path}") csvB = self.browser.find_element_by_id("csvButton") csvB.click() self.browser.find_element_by_id("button-1006").click() self.browser.switch_to.window(self.browser.window_handles[1]) sleep(5) self.logger.debug(f"Ending session")
print('Waiting for elements to load...') s.driver.ensure_element_by_class_name( "desktop-onboarding-sign-up__form-toggler", state='visible').click() if reddit_user_name: s.driver.ensure_element_by_id('user_login').send_keys(reddit_user_name) s.driver.ensure_element_by_id('passwd_login').send_keys(Keys.BACKSPACE) print('Please log-in in the chrome browser') s.driver.ensure_element_by_class_name("desktop-onboarding__title", timeout=60, state='invisible') print('Thanks!') if not reddit_user_name: reddit_user_name = s.driver.xpath( "//span[@class='user']//text()").extract_first() if reddit_user_name: s.transfer_driver_cookies_to_session() response = s.get( "https://www.reddit.com/user/{}/".format(reddit_user_name)) cmnt_karma = response.xpath( "//span[@class='karma comment-karma']//text()").extract_first() reddit_golds_given = response.re_first(r"(\d+) gildings given out") print("Comment karma: {}".format(cmnt_karma)) print("Reddit golds given: {}".format(reddit_golds_given)) else: print("Couldn't get user name")
def gen(num, limit): s = Session(webdriver_path='chromedriver.exe', browser='chrome') s.driver.get("https://privacy.com/login") time.sleep(3) s.driver.find_element_by_xpath( '//*[@id="steps"]/div/form/div[2]/label[1]/input').send_keys( config['username']) s.driver.find_element_by_xpath( '//*[@id="steps"]/div/form/div[2]/label[2]/input').send_keys( config['password']) time.sleep(1) s.driver.find_element_by_xpath( '//*[@id="steps"]/div/form/div[3]/button').click() time.sleep(2) s.transfer_driver_cookies_to_session() s.driver.quit() url1 = "https://privacy.com/api/v1/card" for i in range(int(num)): h1 = { 'Accept': 'application/json, text/plain, */*', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'en-US,en;q=0.9', 'Authorization': 'Bearer {}'.format(s.cookies['token']), 'Connection': 'keep-alive', 'Content-Type': 'application/json;charset=UTF-8', 'Cookie': 'sessionID={}; ETag="ps26i5unssI="; waitlist_cashback=%7B%22refByCode%22%3A%22favicon.ico%22%2C%22isPromotional%22%3Afalse%7D; landing_page=extension-rewards-landing; token={}' .format(s.cookies['sessionID'], s.cookies['token']), 'Host': 'privacy.com', 'Origin': 'https://privacy.com', 'Pragma': 'no-cache', 'Referer': 'https://privacy.com/home', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36' } pay = { "type": "MERCHANT_LOCKED", "spendLimitDuration": "MONTHLY", "memo": genName(), "meta": { "hostname": "" }, "style": 'null', "spendLimit": int(limit), "reloadable": 'true' } r = s.post(url1, json=pay, headers=h1) if r.status_code == requests.codes.ok: print("[{}] !~Created Card~!".format(r.json()['card']['cardID'])) with open("cards.txt", "a+") as file: file.write("{}:{}/{}:{}\n".format(r.json()['card']['pan'], r.json()['card']['expMonth'], r.json()['card']['expYear'], r.json()['card']['cvv'])) else: print("Error Creating Card")
class Driver(object): def __init__(self): # 使用requestium的Session, 使用requests和Selenium, 设置为headless模式 self.s = Session( webdriver_path='./chromedriver', browser='chrome', default_timeout=15, #webdriver_options={'arguments': ['headless']} ) self.category_mapping = None path = os.path.join(os.getcwd(), FILENAME) if os.path.exists(path): self.category_mapping = ujson.load(open(path)) #pprint(self.category_mapping) def close(self): if self.s.driver is not None: self.s.driver.quit() if self.s is not None: self.s.close() def login(self): """ 使用driver登录到启信宝 """ login_url = 'http://www.qixin.com/auth/login?return_url=%2F' self.s.driver.get(login_url) # 使用requestium中的ensure_*方法定位元素 username_xpath = '//input[@class="form-control input-lg input-flat input-flat-user"]' user_element = self.s.driver.ensure_element_by_xpath(username_xpath) for c in USERNAME: # 间歇输入Username和Password user_element.send_keys(c) time.sleep(random.randint(0, 2)) password_xpath = '//input[@class="form-control input-lg input-flat input-flat-lock"]' password_element = self.s.driver.ensure_element_by_xpath( password_xpath) for c in PASSWORD: password_element.send_keys(c) time.sleep(random.random()) password_element.send_keys(Keys.ENTER) self.s.driver.implicitly_wait(10) def process_cookies(self): """ 使用requests抓取页面 """ # 将driver的cookies转给requests的session tmp_url = 'http://www.qixin.com/search?area.province=12&page=1&scope[]=1' self.s.driver.get(tmp_url) self.s.transfer_driver_cookies_to_session() self.s.copy_user_agent_from_driver() # 判断category mapping是否存在 if self.category_mapping is None: req = self.s.get('http://www.qixin.com') self.category_mapping = {} for element in req.xpath('//div[@class="grid-item"]'): category_l1 = element.xpath( './div/text()').extract_first().strip() category_l2 = element.xpath('./a/text()').extract() self.category_mapping[category_l1] = category_l2 ujson.dump(self.category_mapping, open(os.path.join(os.getcwd(), FILENAME), 'w')) def fetch_page(self): # 获取cookies之后,使用requests的session开始抓取数据 result = [] self.s.proxies.update({ 'http': 'http://forward.xdaili.cn:80', 'https': 'https://forward.xdaili.cn:80' }) for page in range(1, 11): url = 'http://www.qixin.com/search?area.province=12&page=%s&scope[]=1&sorter=4' % page self.s.headers.update({'Proxy-Authorization': sign()}) req = self.s.get(url) for element in req.xpath( "//div[contains(@class, 'company-item')]"): result.append({ 'title': element.xpath(".//div[@class='company-title']/a/text()" ).extract_first().strip(), 'legal_owner': element.xpath(".//div[@class='legal-person'][1]/text()" ).re_first(r'法定代表人:(\w*)').strip(), 'status': element.xpath( ".//div[@class='company-tags']/span[1]/text()"). extract_first().strip(), 'capital': element.xpath(".//div[contains(@class, 'col-3-1')]/text()" ).extract_first().strip(), 'date': element.xpath(".//div[contains(@class, 'col-3-2')]/text()" ).extract_first().strip(), 'url': element.xpath(".//div[@class='company-title']/a/@href" ).extract_first().strip() }) time.sleep(10) return result def process_search_condition(self): """ 构建搜索条件 * URL: http://www.qixin.com/search? * param 地区: area.province=12, area.district=120101-120119 * param 搜索范围: scope[]=1 * param 排序: sorter=3 | 4 * param 注册资本: capital: 1-5 * param 所属行业: industry.l1 一级行业, industry.l2 二级行业 * param 注册年份: year: 1-5 * param page: 页码,最大不超过500, 只能看5000条搜索结果 http://www.qixin.com/search?area.district=120101&area.province=12&capital=2&industry.l1=%E5%86%9C%E3%80%81%E6%9E%97%E3%80%81%E7%89%A7%E3%80%81%E6%B8%94%E4%B8%9A&industry.l2=%E5%86%9C%E4%B8%9A&page=1&scope[]=1&sorter=4&year=5 """ pass
class Driver(object): def __init__(self): # 使用requestium的Session, 使用requests和Selenium, 设置为headless模式 self.s = Session( webdriver_path='./chromedriver', browser='chrome', default_timeout=15, #webdriver_options={'arguments': ['headless']} ) # self.category_mapping = None # path = os.path.join(os.getcwd(), FILENAME) # if os.path.exists(path): # self.category_mapping = ujson.load(open(path)) # pprint(self.category_mapping) def close(self): if self.s.driver is not None: self.s.driver.quit() if self.s is not None: self.s.close() def login(self): """ 使用driver登录到启信宝 """ login_url = 'http://www.qixin.com/auth/login?return_url=%2F' self.s.driver.get(login_url) # 使用requestium中的ensure_*方法定位元素 user_element = self.s.driver.ensure_element_by_xpath( LOGIN_XPATH['username']) for c in USERNAME: # 间歇输入Username和Password user_element.send_keys(c) time.sleep(random.randint(0, 2)) password_element = self.s.driver.ensure_element_by_xpath( LOGIN_XPATH['password']) for c in PASSWORD: password_element.send_keys(c) time.sleep(random.random()) password_element.send_keys(Keys.ENTER) self.s.driver.implicitly_wait(20) def process_cookies(self): """ 使用requests抓取页面 """ # 将driver的cookies转给requests的session tmp_url = 'http://www.qixin.com/search?area.province=12&page=1&scope[]=1' self.s.driver.get(tmp_url) self.s.transfer_driver_cookies_to_session() self.s.copy_user_agent_from_driver() # 判断category mapping是否存在 if self.category_mapping is None: req = self.s.get('http://www.qixin.com') self.category_mapping = {} for element in req.xpath(CATEGORY_XPATH['info']): category_l1 = element.xpath( CATEGORY_XPATH['l1']).extract_first().strip() category_l2 = element.xpath(CATEGORY_XPATH['l2']).extract() self.category_mapping[category_l1] = category_l2 ujson.dump(self.category_mapping, open(os.path.join(os.getcwd(), FILENAME), 'w')) def fetch_page_with_chrome(self, url): self.s.transfer_session_cookies_to_driver() self.s.driver.get(url) def fetch_page_with_requests(self, url): """ url = 'http://www.qixin.com/search?area.province=12&page=%s&scope[]=1&sorter=4' % page :param url:请求的URL :param return: 返回list """ # 获取cookies之后,使用requests的session开始抓取数据 self.s.proxies.update({ 'http': 'http://forward.xdaili.cn:80', 'https': 'https://forward.xdaili.cn:80' }) self.s.headers.update({'Proxy-Authorization': sign()}) req = self.s.get(url) result = parse_list(req) return result
class Github: def __init__(self, proxy=None): self.cookies = None self.sess = Session("/usr/local/phantomjs", "phantomjs", default_timeout=15) if proxy: self.sess.proxies['https'] = proxy self.sess.proxies['http'] = proxy self.proxy = proxy self.sess = Session(webdriver_path='/usr/local/bin/chromedriver', browser='phantomjs', default_timeout=15, webdriver_options={'arguments': ['headless']}) if proxy: self.proxies['http'] = proxy self.proxies['https'] = proxy self.user = None def save_session(self, name, password, cookie): gprint("save cred and session") with open(GITHUB_LOGIN, "wb") as fp: u = {"user": name, "pass": password} pickle.dump(u, fp) with open(GITHUB_SESSION, 'wb') as fp: pickle.dump(cookie, fp) def load_session(self): gprint("load seesion form github") if os.path.exists(GITHUB_SESSION): with open(GITHUB_SESSION, 'rb') as fp: self.cookies = pickle.load(fp) self.sess.cookies.update(self.cookies) self.sess.get("https://github.com") self.sess.transfer_session_cookies_to_driver() with open(GITHUB_LOGIN, 'rb') as fp: u = pickle.load(fp) self.user = u['user'] elif os.path.exists(GITHUB_LOGIN): with open(GITHUB_LOGIN, 'rb') as fp: u = pickle.load(fp) self.login(name=u['user'], password=u['pass']) else: name = input('Github name:') passwd = getpass.getpass("Github pass:"******"https://github.com/login") self.sess.driver.find_element_by_css_selector( "input[name=login]").send_keys(name) self.sess.driver.find_element_by_css_selector( "input[name=password]").send_keys(password) self.sess.driver.find_element_by_css_selector( "input[name=commit]").click() self.sess.transfer_driver_cookies_to_session() self.cookies = self.sess.cookies.get_dict() gprint(str(self.cookies)) self.save_session(name, password, self.cookies) def weak_search(self, key): self.load_session() self.search(key, "smtp") self.search(key, "ssh") # with ThreadPoolExecutor(max_workers=10) as exe: # # for k in ['smtp', 'ssh', 'email']: # # s1 = exe.submit(self.search,key, k) # s1.add_done_callback(print) def search(self, *key): gprint(key[-1]) if not self.cookies: self.load_session() res = requests.get("https://github.com/{}/product".format(self.user)) self.cookies = res.cookies.get_dict() gprint(str(self.cookies)) url = "https://github.com/search?q={}&type=code".format("+".join(key)) self.sess.driver.get(url) res = self.sess.driver.page_source b = BeautifulSoup(res, 'lxml') codes = b.select(".code-list-item") if len(codes) > 0: gprint("Found : %d" % len(codes)) else: gprint("Not found:") rprint(b.text.replace("\n", "")) # for i in b.select("a"): # gprint(str(i)) ss = {} for code in codes: k = code.select(".text-bold")[0].text v = { colored(str(n), 'green'): i.text.replace("\n", "") for n, i in enumerate(code.select("td.blob-code")) } gprint(colored(k, "blue")) Tprint(v)
class Charme(object): def __init__(self): # Create a session and authenticate self._s = Session( webdriver_path='/usr/lib/chromium-browser/chromedriver', browser='chrome') #, #webdriver_options={"arguments": ["--headless"]}) self._s.headers.update({ 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:63.0) Gecko/20100101 Firefox/63.0' }) # Login r = self._s.post('https://www.adopteunmec.com/auth/login', data={ 'username': '******', 'password': '******' }) if not r.ok: raise 'Something wrong in login' else: time.sleep(2) def search_by_distance(self, age_min=20, age_max=30, distance=40, sex=1): return self.search({ 'age[min]': age_min, 'age[max]': age_max, 'by': 'distance', 'distance[max]': distance, "sex": sex }) def search(self, criteria=None): if criteria is None: return [] # Go to search page self._s.get('https://www.adopteunmec.com/mySearch') time.sleep(1) # POST a request r = self._s.post('https://www.adopteunmec.com/mySearch/save', data=criteria) time.sleep(3) # Wait a bit... # Trasnfer cookie to selenium, refresh the page, scroll to end 10 times, and get profiles self._s.transfer_session_cookies_to_driver() self._s.driver.get('https://www.adopteunmec.com/mySearch/results') for i in range(10): self._s.driver.find_element_by_tag_name('html').send_keys(Keys.END) time.sleep(.1) html = BeautifulSoup( self._s.driver.execute_script("return document.body.innerHTML"), 'lxml') self._s.transfer_driver_cookies_to_session() self._s.driver.close() # Might be done before ? # Look for <div> tags containing user info blocks = html.find_all('div', {'class': 'user-infos'}) # Get all <a> tags in a same list all_a = [a for sl in [b.find_all('a') for b in blocks] for a in sl] # Extract profiles ID doing common name checks to avoid visit too may profiles later profiles = [ l.get('href').split('/')[-1] for l in all_a if isinstance(l.get('href'), str) and l.get('href').find('profile') > 0 and len(l.get_text()) > 2 ] return profiles def charme(self, profiles=[], max_p=10, filename='data/charme.json'): db = {} try: with open(filename, 'r') as in_f: db = json.load(in_f) except: pass visited = 0 for uid in profiles: # Check if profile already in db if uid not in db: if max_p is not None and visited >= max_p: break visited += 1 url = "https://www.adopteunmec.com/profile/" + uid print "Visiting", url page = self._s.get(url) html = BeautifulSoup( page.content.decode('utf-8', 'xmlcharrefreplace'), 'lxml') img_url = html.find(id='img-current-pic')['src'] img_name = img_url.split('/')[-1] date = datetime.datetime.now().strftime("%m-%d %H:%M") db[uid] = { "profile": url, "name": html.find('div', { 'class': 'username' }).get_text(), "img": img_name, "age": html.find('span', { 'class': 'age' }).get_text(), "city": html.find('span', { 'class': 'city' }).get_text(), "desc": html.find(text='Description').find_parent('div').find( 'p').get_text(), "shop": html.find(text='Shopping List').find_parent('div').find( 'p').get_text(), "charmed": date } # Download and save profile pic pic = self._s.get(img_url, stream=True) pic.raw.decode_content = True with open("data/pics/" + img_name, 'wb') as f: shutil.copyfileobj(pic.raw, f) time.sleep(20) # Bit of rest... # Send a charme url = "https://www.adopteunmec.com/events/charm?id=" + uid r = self._s.get(url) if r.json()['member']['id'] != uid: raise 'Something wrong in response' # Write back json json_s = json.dumps( db) # Dump as a string, to write to file and as JS var with open(filename, 'w') as out_f: out_f.write(json_s)