class ChromeTestCase(unittest.TestCase): def setUp(self): self.s = Session( 'chromedriver', browser='chrome', default_timeout=15, webdriver_options={'arguments': ['headless', 'disable-gpu']}) def test_cookie_transfer_to_requests(self): """Tested on http://testing-ground.scraping.pro/login""" self.s.driver.get('http://testing-ground.scraping.pro/login') self.s.driver.find_element_by_id('usr').send_keys('admin') self.s.driver.ensure_element_by_id('pwd').send_keys( '12345', Keys.ENTER) self.s.driver.ensure_element_by_xpath( '//div[@id="case_login"]/h3[@class="success"]') self.s.transfer_driver_cookies_to_session() response = self.s.get( 'http://testing-ground.scraping.pro/login?mode=welcome') success_message = response.xpath( '//div[@id="case_login"]/h3[@class="success"]/text()' ).extract_first() self.assertEqual( success_message, 'WELCOME :)', 'Failed to transfer cookies from Selenium to Requests') def test_cookie_transfer_to_selenium(self): self.s.get('http://testing-ground.scraping.pro/login') self.s.cookies.set('tdsess', 'TEST_DRIVE_SESSION', domain='testing-ground.scraping.pro') self.s.transfer_session_cookies_to_driver() self.s.driver.get( 'http://testing-ground.scraping.pro/login?mode=welcome') success_message = self.s.driver.xpath( '//div[@id="case_login"]/h3[@class="success"]/text()' ).extract_first() self.assertEqual( success_message, 'WELCOME :)', 'Failed to transfer cookies from Requests to Selenium') def tearDown(self): self.s.driver.close()
class ChromeTestCase(unittest.TestCase): def setUp(self): self.s = Session('chromedriver', browser='chrome', default_timeout=15, webdriver_options={'arguments': ['headless', 'disable-gpu']}) def test_cookie_transfer_to_requests(self): """Tested on http://testing-ground.scraping.pro/login""" self.s.driver.get('http://testing-ground.scraping.pro/login') self.s.driver.find_element_by_id('usr').send_keys('admin') self.s.driver.ensure_element_by_id('pwd').send_keys('12345', Keys.ENTER) self.s.driver.ensure_element_by_xpath('//div[@id="case_login"]/h3[@class="success"]') self.s.transfer_driver_cookies_to_session() response = self.s.get('http://testing-ground.scraping.pro/login?mode=welcome') success_message = response.xpath( '//div[@id="case_login"]/h3[@class="success"]/text()').extract_first() self.assertEqual( success_message, 'WELCOME :)', 'Failed to transfer cookies from Selenium to Requests') def test_cookie_transfer_to_selenium(self): self.s.get('http://testing-ground.scraping.pro/login') self.s.cookies.set('tdsess', 'TEST_DRIVE_SESSION', domain='testing-ground.scraping.pro') self.s.transfer_session_cookies_to_driver() self.s.driver.get('http://testing-ground.scraping.pro/login?mode=welcome') success_message = self.s.driver.xpath( '//div[@id="case_login"]/h3[@class="success"]/text()').extract_first() self.assertEqual( success_message, 'WELCOME :)', 'Failed to transfer cookies from Requests to Selenium') def tearDown(self): self.s.driver.close()
class AuM(object): def __init__(self): # Create a session and authenticate self._s = Session( webdriver_path='/usr/lib/chromium-browser/chromedriver', browser='chrome', webdriver_options={"arguments": ["--headless"]}) self._s.headers.update({ 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:63.0) Gecko/20100101 Firefox/63.0' }) self._s.get('https://www.adopteunmec.com') # Maybe not needed # Register a new account rand_s = ''.join( random.choice(string.ascii_lowercase + string.digits) for _ in range(6)) print('email used: francois_%[email protected]' % rand_s) r = self._s.post('https://www.adopteunmec.com/register/index', data={ 'sex': '1', 'day': '03', 'month': '4', 'year': '1997', 'email': '*****@*****.**' % rand_s, 'password': '******', 'password_check': 'Adottami1', 'country': 'fr', 'zipcode': '06000', 'city': 'Nice', 'confirm_city': '0', 'pseudo': 'RedoAA', 'cgu': '1', 'reg_submit': '', 'by_popup': '1', 'PreventChromeAutocomplete': '' }, headers={ "X-Requested-With": "XMLHttpRequest", "Origin": "https://www.adopteunmec.com/", "Referer": "https://www.adopteunmec.com/" }) status = r.json() # If registration were successful, go to redirect page to confirm account if (status['success'] == 1): self._s.get(status['redirect']) else: print('Something went wrong....') self._common_names = ( 'loic', 'marc', 'anthony', 'tom', 'jordan', 'florian', 'jean', 'manu', 'seb', 'alex', 'lilian', 'angelo', 'fred', 'valent', 'fabrice', 'fabien', 'nico', 'thomas', 'sylvain', 'tim', 'karim', 'robin', 'pierre', 'arnaud', 'max', 'luc', 'mike', 'yann', 'oliv', 'yvan', 'jerem', 'michel', 'mat', 'kev', 'damien', 'vinc', 'eric', 'gilles', 'jona', 'bruno', 'simon', 'adri', 'serge', 'tony', 'jul', 'quentin', 'leo', 'step', 'gab', 'david', 'paul', 'killian', 'alvaro', 'ronan', 'anto', 'jb', 'jp', 'jon', 'patrick', 'virgile', 'juju', 'stef', 'franck', 'alan', 'alain', 'albin', 'alban', 'fran', 'cyril', 'laure', 'phil', 'jacques', 'jack', 'ludo', 'chris', 'vic', 'jo', 'charles', 'geoffrey', 'igor', 'ciro', 'erwan', 'fabio', 'guillaume', 'thibaut', 'romain', 'rafa', 'lionel', 'cedric', 'xavier') def _common_name(self, name): return len( filter(lambda x: x is False, map(lambda n: name.lower().find(n) < 0, self._common_names))) > 0 def search_by_region(self, age_min=20, age_max=30, region=1, sex=0): return self.search({ 'age[min]': age_min, 'age[max]': age_max, 'by': 'region', 'region': region, "sex": sex }) def search_by_disance(self, age_min=20, age_max=30, distance=40, sex=0): return self.search({ 'age[min]': age_min, 'age[max]': age_max, 'by': 'distance', 'distance[max]': distance, "sex": sex }) def search(self, criteria=None): if criteria is None: return [] # Go to search page self._s.get('https://www.adopteunmec.com/mySearch') # POST a request r = self._s.post('https://www.adopteunmec.com/mySearch/save', data=criteria) time.sleep(3) # Wait a bit... # Trasnfer cookie to selenium, refresh the page, scroll to end 10 times, and get profiles self._s.transfer_session_cookies_to_driver() self._s.driver.get('https://www.adopteunmec.com/mySearch/results') for i in range(10): self._s.driver.find_element_by_tag_name('html').send_keys(Keys.END) time.sleep(.1) html = BeautifulSoup( self._s.driver.execute_script("return document.body.innerHTML"), 'lxml') self._s.transfer_driver_cookies_to_session() self._s.driver.close() # Might be done before ? # Look for <div> tags containing user info blocks = html.find_all('div', {'class': 'user-infos'}) # Get all <a> tags in a same list all_a = [a for sl in [b.find_all('a') for b in blocks] for a in sl] # Extract profiles ID doing common name checks to avoid visit too may profiles later profiles = [ l.get('href').split('/')[-1] for l in all_a if isinstance(l.get('href'), str) and l.get('href').find('profile') > 0 and len(l.get_text()) > 2 and not self._common_name(l.get_text()) ] return profiles def update_db(self, profiles=[], max_p=None, filename='data/justemenemoi.json'): db = {} try: with open(filename, 'r') as in_f: db = json.load(in_f) except: pass visited = 0 for uid in profiles: # Check if profile already in db if uid not in db: if max_p is not None and visited >= max_p: break visited += 1 url = "https://www.adopteunmec.com/profile/" + uid page = self._s.get(url) html = BeautifulSoup( page.content.decode('utf-8', 'xmlcharrefreplace')) name = html.find('div', {'class': 'username'}).get_text() desc = html.find(text='Description').find_parent('div').find( 'p').get_text() shop = html.find(text='Shopping List').find_parent('div').find( 'p').get_text() # Profile Filtering if desc.find("non renseign") >= 0 or shop.find( "non renseign") >= 0 or len(desc) < 20 or len( shop) < 20: continue img_url = html.find(id='img-current-pic')['src'] img_name = img_url.split('/')[-1] db[uid] = { "profile": url, "name": name, "img": img_name, "age": html.find('span', { 'class': 'age' }).get_text(), "city": html.find('span', { 'class': 'city' }).get_text(), "desc": desc, "shop": shop } # Download and save profile pic pic = self._s.get(img_url, stream=True) pic.raw.decode_content = True with open("data/pics/" + img_name, 'wb') as f: shutil.copyfileobj(pic.raw, f) time.sleep(.5) # Bit of rest... # Write back json json_s = json.dumps( db) # Dump as a string, to write to file and as JS var with open(filename, 'w') as out_f: out_f.write(json_s) with open(filename + '.js', 'w') as out_f: out_f.write("data = ") out_f.write(json_s)
class Downloader(): def __init__(self, username, password, driver_path=None, download_path=None, headless=True, logger=None): if not logger: logging.basicConfig(level=logging.DEBUG) self.logger = logging.getLogger(__name__) self.logger.setLevel('DEBUG') else: self.logger = logger self._username = username self._password = password self.driver_path = driver_path self.download_path = download_path self.logger = logging.getLogger('odigo_downloader.downloader') self.url = 'https://enregistreur.prosodie.com/odigo4isRecorder/EntryPoint?serviceName=LoginHandler' self.headless = headless self.validated = False self.active = False def __str__(self): return f"\nDOWNLOAD PATH: {self.download_path}\nOPTIONS: {self.webdriver_options}\n" \ f"DRIVER PATH: {self.driver_path}\nUSERNAME: {self._username}\nURL: {self.url}" def setup_selenium_browser(self): if self.active: return f"Session/Browser already active. Cannot have two concurrent sessions/browsers" options = webdriver.ChromeOptions() prefs = { 'download.default_directory': self.download_path, 'download.prompt_for_download': False, 'download.directory_upgrade': True, 'safebrowsing.enabled': False, 'safebrowsing.disable_download_protection': True } options.add_experimental_option('prefs', prefs) if self.headless: options.add_argument('--headless') self.browser = webdriver.Chrome(self.driver_path, options=options) if self.headless: self.browser.command_executor._commands["send_command"] = ( "POST", '/session/$sessionId/chromium/send_command') params = { 'cmd': 'Page.setDownloadBehavior', 'params': { 'behavior': 'allow', 'downloadPath': self.download_path } } command_result = self.browser.execute("send_command", params) for key in command_result: self.logger.debug("result:" + key + ":" + str(command_result[key])) self.active = True def setup_requestium_session(self): if self.active: return f"Session/Browser already active. Cannot have two concurrent sessions/browsers" if self.headless: webdriver_options = {'arguments': ['headless']} else: webdriver_options = {} self.logger.debug( f"Creating Session object with values: {webdriver_options}") self.session = Session(webdriver_path=self.driver_path, browser='chrome', default_timeout=15, webdriver_options=webdriver_options) self.active = True def login_requestium(self): if self.active: raise CustomException(f"Cannot have two active sessions/browsers") self.setup_requestium_session() self.logger.debug(f"Going to URL: {self.url}") self.session.driver.get(self.url) self.logger.debug(f"Entering credentials") self.session.driver.ensure_element_by_name('mail').send_keys( self._username) self.session.driver.ensure_element_by_name('password').send_keys( self._password) self.session.driver.ensure_element_by_name('valider').click() self.validated = True def login_selenium(self): if self.active: raise CustomException(f"Cannot have two active sessions/browsers") self.setup_selenium_browser() self.browser.get(self.url) self.browser.find_element_by_name('mail').send_keys(username) self.browser.find_element_by_name('password').send_keys(password) self.browser.find_element_by_name('valider').click() return def download_mp3(self, path=None, ref=None, xpath=None): self.logger.info( f"\ndownload_mp3 called with:\nPATH: {path},\nREF: {ref},\nXPATH: {xpath}" ) if ref is not None and xpath is None: self.session.driver.ensure_element_by_class_name( 'x-action-col-icon').click() elif xpath is not None and ref is None: self.session.driver.ensure_element_by_xpath(xpath).click() else: self.logger.error("Cannot use both reference number and xpath") return self.session.driver.switch_to.frame('result_frame') time.sleep(1) # Get URL of mp3 file src = self.session.driver.ensure_element_by_id( 'messagePlayer').get_attribute('src') # Selenium --> Requests self.session.transfer_driver_cookies_to_session() # Download r = self.session.get(src, stream=True) if path is None: if ref is None: # Get ref number soap = BeautifulSoup(self.session.driver.page_source, 'lxml') ref = soap.findAll('div', class_='x-grid-cell-inner')[1].text path = '%s.mp3' % ref if r.status_code == 200: with open(path, 'wb') as f: for chunk in r.iter_content(1024 * 2014): f.write(chunk) else: return 1 # Requests --> Selenium self.session.transfer_session_cookies_to_driver() self.session.driver.switch_to.default_content() return def download_mp3_by_ref(self, ref, path=None): self.login_requestium() self.search_by_ref(ref) result = self.download_mp3(path, ref) if result == 1: return 1 self.session.driver.close() def download_mp3_by_csv(self, csv_path, download_dir=None): if download_dir is None: download_dir = self.download_path self.login_requestium() refs = pd.read_csv(csv_path, sep=';').Name length = len(refs) for i, ref in enumerate(refs): sys.stdout.write('\r') sys.stdout.write('downloading: %s/%s' % (i + 1, length)) sys.stdout.flush() self.search_by_ref(ref) mp3_path = None if download_dir is not None: file_name = '%s.mp3' % ref mp3_path = os.path.join(download_dir, file_name) result = self.download_mp3(path=mp3_path, ref=ref) if result == 1: return 1 sys.stdout.write('\n') sys.stdout.flush() self.session.driver.close() return "Finished" def search_by_ref(self, ref): self.session.driver.get(self.url) self.session.driver.ensure_element_by_name('refEr').send_keys(ref) self.session.driver.ensure_element_by_id('button-1009').click() def change_date_format(self, date): try: correct_string = date.strptime(str(date.date()), '%Y-%m-%d').strftime('%d-%m-%Y') return correct_string except Exception as e: raise e def change_time_format(self, date): try: correct_string = date.strptime( str(date.hour) + ':' + str(date.minute), "%H:%M").strftime("%I:%M %p") if correct_string[0] == "0": return correct_string[1::] else: return correct_string except Exception as e: raise e def ceil_dt(self, dt, delta): """Round up to the nearest half hour""" return dt + (datetime.datetime.min - dt) % delta def set_range(self, now): """ Takes current datetime and finds the nearest, previous half hour. Returns the appropriate start and end times and date """ # Format: '10-19-2018' # Format: '12:00 AM' hour_ago = now - datetime.timedelta(minutes=60) rounded = self.ceil_dt(hour_ago, datetime.timedelta(minutes=30)) start_date = self.change_date_format(rounded) start_time = self.change_time_format(rounded) thirty_mins = datetime.timedelta(minutes=30) end_date = start_date end_time = self.change_time_format(rounded + thirty_mins) return (start_date, start_time, end_date, end_time) def search_by_range(self, start_date, start_time, end_date, end_time): """ Doesn't work correctly. Date seems to work but time not so much. Search records on www.prosodie.com by date range and return session. Input: s -- Requestium session (required | type: requestium.requestium.Session); start_date -- start date (not required | type: str). Format: 'mm:dd:yyyy'. Example: '03-05-1991'; start_time -- start time (not required | type: str). Example: '12:00 AM'; end_date -- end date (not required | type: str). Format: 'mm:dd:yyyy'. Example: '03-05-1991'; end_time -- end time (not required | type: str). Example: '12:00 PM'. Output: s -- Requestium session (type: requestium.requestium.Session). """ if start_date: self.browser.find_element_by_name('dateDebut').send_keys( start_date) if start_time: self.browser.find_element_by_name('heureDebut').send_keys( start_time) if end_date: self.browser.find_element_by_name('dateFin').send_keys(end_date) if end_time: self.browser.find_element_by_name('heureFin').send_keys(end_time) self.browser.find_element_by_id('button-1009').click() return def download_all_half_hour(self): self.logger.debug(f"Downloading calls from last half hour") self.logger.debug(f"Login check...") if not self.validated: self.logger.debug(f"Not logged in. Validating") self.login_selenium() self.logger.debug(f"Logged in.") self.logger.debug(f"Getting search range") search_range = self.set_range(datetime.datetime.now()) sleep(2) self.logger.debug(f"Applying filters") self.browser.find_element_by_id("criteres-inputEl").send_keys('_EN') self.search_by_range(*search_range) sleep(5) self.logger.debug(f"Downloading results to {self.download_path}") csvB = self.browser.find_element_by_id("csvButton") csvB.click() self.browser.find_element_by_id("button-1006").click() self.browser.switch_to.window(self.browser.window_handles[1]) sleep(5) self.logger.debug(f"Ending session")
class Driver(object): def __init__(self): # 使用requestium的Session, 使用requests和Selenium, 设置为headless模式 self.s = Session( webdriver_path='./chromedriver', browser='chrome', default_timeout=15, #webdriver_options={'arguments': ['headless']} ) # self.category_mapping = None # path = os.path.join(os.getcwd(), FILENAME) # if os.path.exists(path): # self.category_mapping = ujson.load(open(path)) # pprint(self.category_mapping) def close(self): if self.s.driver is not None: self.s.driver.quit() if self.s is not None: self.s.close() def login(self): """ 使用driver登录到启信宝 """ login_url = 'http://www.qixin.com/auth/login?return_url=%2F' self.s.driver.get(login_url) # 使用requestium中的ensure_*方法定位元素 user_element = self.s.driver.ensure_element_by_xpath( LOGIN_XPATH['username']) for c in USERNAME: # 间歇输入Username和Password user_element.send_keys(c) time.sleep(random.randint(0, 2)) password_element = self.s.driver.ensure_element_by_xpath( LOGIN_XPATH['password']) for c in PASSWORD: password_element.send_keys(c) time.sleep(random.random()) password_element.send_keys(Keys.ENTER) self.s.driver.implicitly_wait(20) def process_cookies(self): """ 使用requests抓取页面 """ # 将driver的cookies转给requests的session tmp_url = 'http://www.qixin.com/search?area.province=12&page=1&scope[]=1' self.s.driver.get(tmp_url) self.s.transfer_driver_cookies_to_session() self.s.copy_user_agent_from_driver() # 判断category mapping是否存在 if self.category_mapping is None: req = self.s.get('http://www.qixin.com') self.category_mapping = {} for element in req.xpath(CATEGORY_XPATH['info']): category_l1 = element.xpath( CATEGORY_XPATH['l1']).extract_first().strip() category_l2 = element.xpath(CATEGORY_XPATH['l2']).extract() self.category_mapping[category_l1] = category_l2 ujson.dump(self.category_mapping, open(os.path.join(os.getcwd(), FILENAME), 'w')) def fetch_page_with_chrome(self, url): self.s.transfer_session_cookies_to_driver() self.s.driver.get(url) def fetch_page_with_requests(self, url): """ url = 'http://www.qixin.com/search?area.province=12&page=%s&scope[]=1&sorter=4' % page :param url:请求的URL :param return: 返回list """ # 获取cookies之后,使用requests的session开始抓取数据 self.s.proxies.update({ 'http': 'http://forward.xdaili.cn:80', 'https': 'https://forward.xdaili.cn:80' }) self.s.headers.update({'Proxy-Authorization': sign()}) req = self.s.get(url) result = parse_list(req) return result
class Github: def __init__(self, proxy=None): self.cookies = None self.sess = Session("/usr/local/phantomjs", "phantomjs", default_timeout=15) if proxy: self.sess.proxies['https'] = proxy self.sess.proxies['http'] = proxy self.proxy = proxy self.sess = Session(webdriver_path='/usr/local/bin/chromedriver', browser='phantomjs', default_timeout=15, webdriver_options={'arguments': ['headless']}) if proxy: self.proxies['http'] = proxy self.proxies['https'] = proxy self.user = None def save_session(self, name, password, cookie): gprint("save cred and session") with open(GITHUB_LOGIN, "wb") as fp: u = {"user": name, "pass": password} pickle.dump(u, fp) with open(GITHUB_SESSION, 'wb') as fp: pickle.dump(cookie, fp) def load_session(self): gprint("load seesion form github") if os.path.exists(GITHUB_SESSION): with open(GITHUB_SESSION, 'rb') as fp: self.cookies = pickle.load(fp) self.sess.cookies.update(self.cookies) self.sess.get("https://github.com") self.sess.transfer_session_cookies_to_driver() with open(GITHUB_LOGIN, 'rb') as fp: u = pickle.load(fp) self.user = u['user'] elif os.path.exists(GITHUB_LOGIN): with open(GITHUB_LOGIN, 'rb') as fp: u = pickle.load(fp) self.login(name=u['user'], password=u['pass']) else: name = input('Github name:') passwd = getpass.getpass("Github pass:"******"https://github.com/login") self.sess.driver.find_element_by_css_selector( "input[name=login]").send_keys(name) self.sess.driver.find_element_by_css_selector( "input[name=password]").send_keys(password) self.sess.driver.find_element_by_css_selector( "input[name=commit]").click() self.sess.transfer_driver_cookies_to_session() self.cookies = self.sess.cookies.get_dict() gprint(str(self.cookies)) self.save_session(name, password, self.cookies) def weak_search(self, key): self.load_session() self.search(key, "smtp") self.search(key, "ssh") # with ThreadPoolExecutor(max_workers=10) as exe: # # for k in ['smtp', 'ssh', 'email']: # # s1 = exe.submit(self.search,key, k) # s1.add_done_callback(print) def search(self, *key): gprint(key[-1]) if not self.cookies: self.load_session() res = requests.get("https://github.com/{}/product".format(self.user)) self.cookies = res.cookies.get_dict() gprint(str(self.cookies)) url = "https://github.com/search?q={}&type=code".format("+".join(key)) self.sess.driver.get(url) res = self.sess.driver.page_source b = BeautifulSoup(res, 'lxml') codes = b.select(".code-list-item") if len(codes) > 0: gprint("Found : %d" % len(codes)) else: gprint("Not found:") rprint(b.text.replace("\n", "")) # for i in b.select("a"): # gprint(str(i)) ss = {} for code in codes: k = code.select(".text-bold")[0].text v = { colored(str(n), 'green'): i.text.replace("\n", "") for n, i in enumerate(code.select("td.blob-code")) } gprint(colored(k, "blue")) Tprint(v)
class Charme(object): def __init__(self): # Create a session and authenticate self._s = Session( webdriver_path='/usr/lib/chromium-browser/chromedriver', browser='chrome') #, #webdriver_options={"arguments": ["--headless"]}) self._s.headers.update({ 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:63.0) Gecko/20100101 Firefox/63.0' }) # Login r = self._s.post('https://www.adopteunmec.com/auth/login', data={ 'username': '******', 'password': '******' }) if not r.ok: raise 'Something wrong in login' else: time.sleep(2) def search_by_distance(self, age_min=20, age_max=30, distance=40, sex=1): return self.search({ 'age[min]': age_min, 'age[max]': age_max, 'by': 'distance', 'distance[max]': distance, "sex": sex }) def search(self, criteria=None): if criteria is None: return [] # Go to search page self._s.get('https://www.adopteunmec.com/mySearch') time.sleep(1) # POST a request r = self._s.post('https://www.adopteunmec.com/mySearch/save', data=criteria) time.sleep(3) # Wait a bit... # Trasnfer cookie to selenium, refresh the page, scroll to end 10 times, and get profiles self._s.transfer_session_cookies_to_driver() self._s.driver.get('https://www.adopteunmec.com/mySearch/results') for i in range(10): self._s.driver.find_element_by_tag_name('html').send_keys(Keys.END) time.sleep(.1) html = BeautifulSoup( self._s.driver.execute_script("return document.body.innerHTML"), 'lxml') self._s.transfer_driver_cookies_to_session() self._s.driver.close() # Might be done before ? # Look for <div> tags containing user info blocks = html.find_all('div', {'class': 'user-infos'}) # Get all <a> tags in a same list all_a = [a for sl in [b.find_all('a') for b in blocks] for a in sl] # Extract profiles ID doing common name checks to avoid visit too may profiles later profiles = [ l.get('href').split('/')[-1] for l in all_a if isinstance(l.get('href'), str) and l.get('href').find('profile') > 0 and len(l.get_text()) > 2 ] return profiles def charme(self, profiles=[], max_p=10, filename='data/charme.json'): db = {} try: with open(filename, 'r') as in_f: db = json.load(in_f) except: pass visited = 0 for uid in profiles: # Check if profile already in db if uid not in db: if max_p is not None and visited >= max_p: break visited += 1 url = "https://www.adopteunmec.com/profile/" + uid print "Visiting", url page = self._s.get(url) html = BeautifulSoup( page.content.decode('utf-8', 'xmlcharrefreplace'), 'lxml') img_url = html.find(id='img-current-pic')['src'] img_name = img_url.split('/')[-1] date = datetime.datetime.now().strftime("%m-%d %H:%M") db[uid] = { "profile": url, "name": html.find('div', { 'class': 'username' }).get_text(), "img": img_name, "age": html.find('span', { 'class': 'age' }).get_text(), "city": html.find('span', { 'class': 'city' }).get_text(), "desc": html.find(text='Description').find_parent('div').find( 'p').get_text(), "shop": html.find(text='Shopping List').find_parent('div').find( 'p').get_text(), "charmed": date } # Download and save profile pic pic = self._s.get(img_url, stream=True) pic.raw.decode_content = True with open("data/pics/" + img_name, 'wb') as f: shutil.copyfileobj(pic.raw, f) time.sleep(20) # Bit of rest... # Send a charme url = "https://www.adopteunmec.com/events/charm?id=" + uid r = self._s.get(url) if r.json()['member']['id'] != uid: raise 'Something wrong in response' # Write back json json_s = json.dumps( db) # Dump as a string, to write to file and as JS var with open(filename, 'w') as out_f: out_f.write(json_s)