class AuM(object): def __init__(self): # Create a session and authenticate self._s = Session( webdriver_path='/usr/lib/chromium-browser/chromedriver', browser='chrome', webdriver_options={"arguments": ["--headless"]}) self._s.headers.update({ 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:63.0) Gecko/20100101 Firefox/63.0' }) self._s.get('https://www.adopteunmec.com') # Maybe not needed # Register a new account rand_s = ''.join( random.choice(string.ascii_lowercase + string.digits) for _ in range(6)) print('email used: francois_%[email protected]' % rand_s) r = self._s.post('https://www.adopteunmec.com/register/index', data={ 'sex': '1', 'day': '03', 'month': '4', 'year': '1997', 'email': '*****@*****.**' % rand_s, 'password': '******', 'password_check': 'Adottami1', 'country': 'fr', 'zipcode': '06000', 'city': 'Nice', 'confirm_city': '0', 'pseudo': 'RedoAA', 'cgu': '1', 'reg_submit': '', 'by_popup': '1', 'PreventChromeAutocomplete': '' }, headers={ "X-Requested-With": "XMLHttpRequest", "Origin": "https://www.adopteunmec.com/", "Referer": "https://www.adopteunmec.com/" }) status = r.json() # If registration were successful, go to redirect page to confirm account if (status['success'] == 1): self._s.get(status['redirect']) else: print('Something went wrong....') self._common_names = ( 'loic', 'marc', 'anthony', 'tom', 'jordan', 'florian', 'jean', 'manu', 'seb', 'alex', 'lilian', 'angelo', 'fred', 'valent', 'fabrice', 'fabien', 'nico', 'thomas', 'sylvain', 'tim', 'karim', 'robin', 'pierre', 'arnaud', 'max', 'luc', 'mike', 'yann', 'oliv', 'yvan', 'jerem', 'michel', 'mat', 'kev', 'damien', 'vinc', 'eric', 'gilles', 'jona', 'bruno', 'simon', 'adri', 'serge', 'tony', 'jul', 'quentin', 'leo', 'step', 'gab', 'david', 'paul', 'killian', 'alvaro', 'ronan', 'anto', 'jb', 'jp', 'jon', 'patrick', 'virgile', 'juju', 'stef', 'franck', 'alan', 'alain', 'albin', 'alban', 'fran', 'cyril', 'laure', 'phil', 'jacques', 'jack', 'ludo', 'chris', 'vic', 'jo', 'charles', 'geoffrey', 'igor', 'ciro', 'erwan', 'fabio', 'guillaume', 'thibaut', 'romain', 'rafa', 'lionel', 'cedric', 'xavier') def _common_name(self, name): return len( filter(lambda x: x is False, map(lambda n: name.lower().find(n) < 0, self._common_names))) > 0 def search_by_region(self, age_min=20, age_max=30, region=1, sex=0): return self.search({ 'age[min]': age_min, 'age[max]': age_max, 'by': 'region', 'region': region, "sex": sex }) def search_by_disance(self, age_min=20, age_max=30, distance=40, sex=0): return self.search({ 'age[min]': age_min, 'age[max]': age_max, 'by': 'distance', 'distance[max]': distance, "sex": sex }) def search(self, criteria=None): if criteria is None: return [] # Go to search page self._s.get('https://www.adopteunmec.com/mySearch') # POST a request r = self._s.post('https://www.adopteunmec.com/mySearch/save', data=criteria) time.sleep(3) # Wait a bit... # Trasnfer cookie to selenium, refresh the page, scroll to end 10 times, and get profiles self._s.transfer_session_cookies_to_driver() self._s.driver.get('https://www.adopteunmec.com/mySearch/results') for i in range(10): self._s.driver.find_element_by_tag_name('html').send_keys(Keys.END) time.sleep(.1) html = BeautifulSoup( self._s.driver.execute_script("return document.body.innerHTML"), 'lxml') self._s.transfer_driver_cookies_to_session() self._s.driver.close() # Might be done before ? # Look for <div> tags containing user info blocks = html.find_all('div', {'class': 'user-infos'}) # Get all <a> tags in a same list all_a = [a for sl in [b.find_all('a') for b in blocks] for a in sl] # Extract profiles ID doing common name checks to avoid visit too may profiles later profiles = [ l.get('href').split('/')[-1] for l in all_a if isinstance(l.get('href'), str) and l.get('href').find('profile') > 0 and len(l.get_text()) > 2 and not self._common_name(l.get_text()) ] return profiles def update_db(self, profiles=[], max_p=None, filename='data/justemenemoi.json'): db = {} try: with open(filename, 'r') as in_f: db = json.load(in_f) except: pass visited = 0 for uid in profiles: # Check if profile already in db if uid not in db: if max_p is not None and visited >= max_p: break visited += 1 url = "https://www.adopteunmec.com/profile/" + uid page = self._s.get(url) html = BeautifulSoup( page.content.decode('utf-8', 'xmlcharrefreplace')) name = html.find('div', {'class': 'username'}).get_text() desc = html.find(text='Description').find_parent('div').find( 'p').get_text() shop = html.find(text='Shopping List').find_parent('div').find( 'p').get_text() # Profile Filtering if desc.find("non renseign") >= 0 or shop.find( "non renseign") >= 0 or len(desc) < 20 or len( shop) < 20: continue img_url = html.find(id='img-current-pic')['src'] img_name = img_url.split('/')[-1] db[uid] = { "profile": url, "name": name, "img": img_name, "age": html.find('span', { 'class': 'age' }).get_text(), "city": html.find('span', { 'class': 'city' }).get_text(), "desc": desc, "shop": shop } # Download and save profile pic pic = self._s.get(img_url, stream=True) pic.raw.decode_content = True with open("data/pics/" + img_name, 'wb') as f: shutil.copyfileobj(pic.raw, f) time.sleep(.5) # Bit of rest... # Write back json json_s = json.dumps( db) # Dump as a string, to write to file and as JS var with open(filename, 'w') as out_f: out_f.write(json_s) with open(filename + '.js', 'w') as out_f: out_f.write("data = ") out_f.write(json_s)
class API: def __init__(self, path): self.last_json = "" self.last_response = None self.IG_SIG_KEY = '4f8732eb9ba7d1c8e8897a75d6474d4eb3f5279137431b2aafb71fafe2abe178' self.SIG_KEY_VERSION = '4' self.USER_AGENT = 'Instagram 10.26.0 Android ({android_version}/{android_release}; 640dpi; 1440x2560; {manufacturer}; {device}; {model}; samsungexynos8890; en_US)'.format( **DEVICE_SETTINTS) self.s = Session(webdriver_path=path, browser='chrome', default_timeout=15) self.logger = logging.getLogger('[instatesi_{}]'.format(id(self))) self.privateUsers = {} self.users = {} fh = logging.FileHandler(filename='instatesi.log') fh.setLevel(logging.INFO) fh.setFormatter(logging.Formatter('%(asctime)s %(message)s')) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter( logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')) self.logger.addHandler(fh) self.logger.addHandler(ch) self.logger.setLevel(logging.DEBUG) self.lastUserHandled = None def saveScrapedFollowers(self): import json self.logger.info("Except for the selected Followers...") if not os.path.exists(os.getcwd() + "/ScrapedFollowers/" + self.lastUserHandled + ".txt"): with open( os.getcwd() + "/ScrapedFollowers/" + self.lastUserHandled + ".txt", "w") as f: """ f.write("Scraped following from " + self.lastUserHandled +"\n") f.write("-------Non private users-------\n") f.write(json.dumps(self.users[''], indent=2)) f.write("\n-------Private users-------\n") f.write(json.dumps(self.privateUsers, indent=2)) """ for k, v in self.users.items(): f.write(k + ',\n') self.logger.info("Following successfully saved!") self.users = dict() self.privateUsers = dict() else: self.logger.warning( "Warning! The user is already present in the database. Overwrite?" ) #Define some logic for file overwriting def saveScrapedFollowing(self): import json self.logger.info("Except for the Following's Following...") if not os.path.exists(os.getcwd() + "/ScrapedFollowing/" + self.lastUserHandled + ".txt"): with open( os.getcwd() + "/ScrapedFollowing/" + self.lastUserHandled + ".txt", "w") as f: f.write("Scraped following from " + self.lastUserHandled + "\n") f.write("-------Non private users-------\n") f.write(json.dumps(self.users, indent=2)) f.write("\n-------Private users-------\n") f.write(json.dumps(self.privateUsers, indent=2)) self.logger.info("Following successfully saved!") self.users = dict() self.privateUsers = dict() else: self.logger.warning( "Warning! The user is already present in the database. Overwrite?" ) #Define some logic for file overwriting def getUserFollowers(self, userID, rank_token, selection="followers"): self.logger.info("User ID follower scraping started " + str(userID)) followers = self.getTotalFollowers(userID, rank_token, fromInput=selection) return [str(item['username']) for item in followers][::-1] if followers else [] def __getUsernameInfo(self, usernameId): return self.__send_request('users/' + str(usernameId) + '/info/') def __send_request_for_user_followers(self, user_id, rank_token, max_id='', selection="followers"): url = 'friendships/{user_id}/followers/?rank_token={rank_token}' if selection == "followers" else 'friendships/{user_id}/following/?max_id={max_id}&ig_sig_key_version={sig_key}&rank_token={rank_token}' url = url.format( user_id=user_id, rank_token=rank_token) if selection == "followers" else url.format( user_id=user_id, max_id=max_id, sig_key=self.SIG_KEY_VERSION, rank_token=rank_token) if max_id: url += '&max_id={max_id}'.format(max_id=max_id) return self.__send_request(url) def searchUsername(self, username): url = 'users/{username}/usernameinfo/'.format(username=username) self.logger.info("Looking for user information " + username) return self.__send_request(url) def getUsernameFromID(self, user_id): url = 'users/{user_id}/info/'.format(user_id=user_id) self.__send_request(url) self.logger.info("Return the requested username, or " + str(self.last_json['user']['username'])) return self.last_json['user']['username'] def __generateSignature(self, data, IG_SIG_KEY, SIG_KEY_VERSION): body = hmac.new( IG_SIG_KEY.encode('utf-8'), data.encode('utf-8'), hashlib.sha256).hexdigest() + '.' + urllib.parse.quote(data) signature = 'ig_sig_key_version={sig_key}&signed_body={body}' return signature.format(sig_key=SIG_KEY_VERSION, body=body) def castUsernameToUserID(self, usernameToLook): self.lastUserHandled = usernameToLook userID = "" self.searchUsername(usernameToLook) if "user" in self.last_json: userID = str(self.last_json["user"]["pk"]) self.logger.info("The username " + usernameToLook + " corresponds to the ID " + userID) return userID def seeStories(self): self.__send_request("feed/reels_tray/") return self.last_json def getTotalFollowers(self, usernameId, rank_token, fromInput="followers"): sleep_track = 0 followers = [] next_max_id = '' self.__getUsernameInfo(usernameId) if "user" in self.last_json: total_followers = self.last_json["user"][ 'follower_count'] if fromInput == "followers" else self.last_json[ "user"]['following_count'] if total_followers > 200000: self.logger.warning( "There are over 200,000 followers. It may take a while.") else: return False with tqdm(total=total_followers, desc="Retrieving followers", leave=False) as pbar: while True: self.__send_request_for_user_followers(usernameId, rank_token, next_max_id, selection=fromInput) temp = self.last_json try: pbar.update(len(temp["users"])) for item in temp["users"]: if item['is_private']: self.privateUsers[item['username']] = { 'ID': item['pk'], 'user_handle': item['username'], 'is_verified': item['is_verified'], 'is_private': item['is_private'], 'profile pic': item['profile_pic_url'], 'Full Name': item['full_name'] } else: self.users[item['username']] = { 'ID': item['pk'], 'user_handle': item['username'], 'is_private': item['is_private'], 'is_verified': item['is_verified'], 'profile pic': item['profile_pic_url'], 'Full Name': item['full_name'] } followers.append(item) sleep_track += 1 if sleep_track >= 20000: import random sleep_time = random.randint(120, 180) self.logger.info("Waiting for " + str(float(sleep_time / 60)) + " due to excessive demands.") time.sleep(sleep_time) sleep_track = 0 if len(temp["users"] ) == 0 or len(followers) >= total_followers: self.logger.info( "Returning account followers in the scraping phase, ie " + str(len(followers[:total_followers]))) return followers[:total_followers] except Exception: self.logger.error( "Returning account followers in the scraping phase, ie " + str(len(followers[:total_followers]))) return followers[:total_followers] if temp["big_list"] is False: self.logger.info( "Returning account followers in the scraping phase, ie " + str(len(followers[:total_followers]))) return followers[:total_followers] next_max_id = temp["next_max_id"] def __send_request(self, endpoint, post=None, login=False, with_signature=True): self.s.headers.update({ 'Connection': 'close', 'Accept': '*/*', 'Content-type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Cookie2': '$Version=1', 'Accept-Language': 'en-US', 'User-Agent': self.USER_AGENT }) try: if post is not None: # POST if with_signature: post = self.__generateSignature(post, self.IG_SIG_KEY, self.SIG_KEY_VERSION) response = self.s.post('https://i.instagram.com/api/v1/' + endpoint, data=post) else: # GET response = self.s.get('https://i.instagram.com/api/v1/' + endpoint) except Exception as e: self.logger.error("Exception due to endpoint " + endpoint) self.logger.error(e) return False if response.status_code == 200: self.logger.info("The request to the endpoint " + endpoint + " has been successful") self.last_response = response self.last_json = json.loads(response.text) return True else: return False
class Monster(): api_throttle_secs = 3 def __init__( self ): self.verbose = False self._session = Session( webdriver_path='' ,browser='chrome' ,default_timeout=15 ,webdriver_options={ 'arguments' : [ 'headless' ] } ) @sleep_and_retry @limits( calls=1, period=api_throttle_secs ) def apply( self, job_link ): '''Apply to the job at the given job link for Monster.com. Args: job_link (str_or_SearchResult): the speed apply link for the job to apply to. Returns: bool: True if successful, False otherwise. ''' if isinstance( job_link, SearchResult ): job_link = job_link.ApplyLink apply_result = self._session.get( job_link ) if apply_result.status_code == 200: if apply_result.json()['success'] == True: return True elif self.verbose: print( job_link ) print( apply_result.json() ) return False def batchApply( self, job_links ): ''' Apply to all jobs in the list of job links given Args: job_links (list_or_generator): List, tuple, or generator of job links Returns: jobs_applied_to (int): The number of jobs applied to successfully ''' jobs_quantity = 0 quantity_applied_to = 0 if not isinstance( job_links, types.GeneratorType ): jobs_quantity = len( job_links ) progress_bar = tqdm( total=jobs_quantity ,desc='Applying' ,unit='Jobs' ) for job_link in job_links: if isinstance( job_links, types.GeneratorType ): progress_bar.total += 1 if self.apply( job_link ): progress_bar.update( 1 ) jobs_applied_to = progress_bar.n return jobs_applied_to @sleep_and_retry @limits( calls=1, period=api_throttle_secs ) def login( self, email, password ): '''Login to the Monster.com job board site. Args: email (str): Email address for logging into Monster.com. password (str): Password corresponding to email address to login to Monster.com job board site. Returns: bool: True if successful, False otherwise. ''' # GOTO LOGIN PAGE TO CHECK IF AVAILABLE & GET COOKIES login_page = self._session.get( SITE['login'] ) if login_page.status_code != 200: raise Exception( 'ERROR: COULD NOT GET LOGIN PAGE FOR MONSTER.COM : ' + SITE['login'] ) # BUILD FORM DATA login_data = { 'AreCookiesEnabled' : True ,'EmailAddress' : email ,'IsComingFromProtectedView': False ,'IsKeepMeLoggedInEnabled' : True ,'Password' : password ,'PersistLogin' : True } request_verification_token = \ login_page.xpath('//input[@name="__RequestVerificationToken"]/@value').extract()[0] login_data.update( { '__RequestVerificationToken' : request_verification_token } ) # LOGIN login_result = self._session.post( SITE['login'], data=login_data ) if login_result.status_code == 200: return True else: return False @sleep_and_retry @limits( calls=1, period=api_throttle_secs ) def getJobDetails( self, job_link ): ''' Get dictionary of details of the job, such as title and description. Args: job_link (str or int): Either a url containing the job id in the format of jobid={}, such as the apply link or the job page link. Or, directly supply the job id if it is available. Returns: job_dict (dict): Dictionary of the job link, job title, company name, job address, and job description. ''' job_link = str( job_link ) if not 'jobid' in job_link: job_id = job_link else: job_id = parse.parse_qs( parse.urlparse( job_link ).query )['jobid'][0] job_url = SITE[ 'job' ].format( job_id ) job_page = self._session.get( job_url ) job_json = job_page.json() job_description = job_json[ 'jobDescription' ] job_title = job_json[ 'companyInfo' ][ 'companyHeader' ] company_name = job_json[ 'companyInfo' ][ 'name' ] job_address = job_json[ 'companyInfo' ][ 'jobLocation' ] job_dict = { 'job_link' : job_link ,'job_title' : job_title ,'job_address' : job_address ,'company_name' : company_name ,'job_description' : job_description } return job_dict def search( self, quantity=25, filter_out_recruiting_agencies=True, **kwargs ): ''' Search Monster.com with the given filters and yield job links. Args: quantity (int): The max number of results to return. kwargs (dict): Dictionary of filters, such as keywords, type (full_time,part_time), and posteddaysago. Returns: SearchResult (namedtuple): generator of named tuples, each containing an ApplyLink and a DetailsLink. The ApplyLink, when followed, will apply for the job automatically. The Details link will return json data about the job. ''' search_url = SITE['search']['root'] # HANDLE SPECIAL CASE OF JOB TYPE, WHICH MUST PRECEED QUERY job_type_value = '' if 'type' in kwargs: job_type = kwargs['type'] options = SITE['search']['type']['options'] job_type_value = options[job_type] if job_type in options else '' kwargs.pop( 'type' ) search_url = search_url.format( type=urllib.parse.quote_plus( job_type_value ) ) # FORMAT URL WITH REMAINING FILTERS for search_field, search_value in kwargs.items(): if search_field in SITE['search']: if isinstance( SITE['search'][search_field], dict ): options = SITE['search'][search_field]['options'] if search_value in options: options_value = options[search_value] search_url += '+' + urllib.parse.quote_plus( options_value ) else: search_format = SITE['search'][search_field] search_url += \ '&{0}'.format(search_format.format(urllib.parse.quote_plus(search_value))) @sleep_and_retry @limits( calls=1, period=self.api_throttle_secs ) def getPage( page ): paged_search_url = search_url + '&page=' + str( page ) search_page = self._session.get( paged_search_url ) return search_page # GET AND PROCESS RETURNED JSON quantity_returned = 0 page = 1 while quantity_returned < quantity: search_page = getPage( page ) if search_page.status_code != 200: break search_json = search_page.json() for app_dict in search_json: if all( key in app_dict for key in ( 'MusangKingId', 'ApplyType' ) ): if app_dict['MusangKingId'] != 0 and app_dict['ApplyType'] != None: # filter jobs that are missing data / poorly formatted if any( x in app_dict['ApplyType'] \ for x in QUICK_APPLY_KEYWORDS ): # filter to include quick apply jobs only if not any( x.lower() in app_dict['Company']['Name'].lower() \ for x in RECRUITING_AGENCY_KEYWORDS ) or \ not filter_out_recruiting_agencies: # filter jobs from recruiting agencies job_id = app_dict['MusangKingId'] apply_url = SITE['speedapply'].format( job_id ) details_url = SITE['job'].format( job_id ) search_result = SearchResult( apply_url, details_url ) quantity_returned += 1 yield search_result if quantity_returned >= quantity: break page += 1
def gen(num, limit): s = Session(webdriver_path='chromedriver.exe', browser='chrome') s.driver.get("https://privacy.com/login") time.sleep(3) s.driver.find_element_by_xpath( '//*[@id="steps"]/div/form/div[2]/label[1]/input').send_keys( config['username']) s.driver.find_element_by_xpath( '//*[@id="steps"]/div/form/div[2]/label[2]/input').send_keys( config['password']) time.sleep(1) s.driver.find_element_by_xpath( '//*[@id="steps"]/div/form/div[3]/button').click() time.sleep(2) s.transfer_driver_cookies_to_session() s.driver.quit() url1 = "https://privacy.com/api/v1/card" for i in range(int(num)): h1 = { 'Accept': 'application/json, text/plain, */*', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'en-US,en;q=0.9', 'Authorization': 'Bearer {}'.format(s.cookies['token']), 'Connection': 'keep-alive', 'Content-Type': 'application/json;charset=UTF-8', 'Cookie': 'sessionID={}; ETag="ps26i5unssI="; waitlist_cashback=%7B%22refByCode%22%3A%22favicon.ico%22%2C%22isPromotional%22%3Afalse%7D; landing_page=extension-rewards-landing; token={}' .format(s.cookies['sessionID'], s.cookies['token']), 'Host': 'privacy.com', 'Origin': 'https://privacy.com', 'Pragma': 'no-cache', 'Referer': 'https://privacy.com/home', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36' } pay = { "type": "MERCHANT_LOCKED", "spendLimitDuration": "MONTHLY", "memo": genName(), "meta": { "hostname": "" }, "style": 'null', "spendLimit": int(limit), "reloadable": 'true' } r = s.post(url1, json=pay, headers=h1) if r.status_code == requests.codes.ok: print("[{}] !~Created Card~!".format(r.json()['card']['cardID'])) with open("cards.txt", "a+") as file: file.write("{}:{}/{}:{}\n".format(r.json()['card']['pan'], r.json()['card']['expMonth'], r.json()['card']['expYear'], r.json()['card']['cvv'])) else: print("Error Creating Card")
class Charme(object): def __init__(self): # Create a session and authenticate self._s = Session( webdriver_path='/usr/lib/chromium-browser/chromedriver', browser='chrome') #, #webdriver_options={"arguments": ["--headless"]}) self._s.headers.update({ 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:63.0) Gecko/20100101 Firefox/63.0' }) # Login r = self._s.post('https://www.adopteunmec.com/auth/login', data={ 'username': '******', 'password': '******' }) if not r.ok: raise 'Something wrong in login' else: time.sleep(2) def search_by_distance(self, age_min=20, age_max=30, distance=40, sex=1): return self.search({ 'age[min]': age_min, 'age[max]': age_max, 'by': 'distance', 'distance[max]': distance, "sex": sex }) def search(self, criteria=None): if criteria is None: return [] # Go to search page self._s.get('https://www.adopteunmec.com/mySearch') time.sleep(1) # POST a request r = self._s.post('https://www.adopteunmec.com/mySearch/save', data=criteria) time.sleep(3) # Wait a bit... # Trasnfer cookie to selenium, refresh the page, scroll to end 10 times, and get profiles self._s.transfer_session_cookies_to_driver() self._s.driver.get('https://www.adopteunmec.com/mySearch/results') for i in range(10): self._s.driver.find_element_by_tag_name('html').send_keys(Keys.END) time.sleep(.1) html = BeautifulSoup( self._s.driver.execute_script("return document.body.innerHTML"), 'lxml') self._s.transfer_driver_cookies_to_session() self._s.driver.close() # Might be done before ? # Look for <div> tags containing user info blocks = html.find_all('div', {'class': 'user-infos'}) # Get all <a> tags in a same list all_a = [a for sl in [b.find_all('a') for b in blocks] for a in sl] # Extract profiles ID doing common name checks to avoid visit too may profiles later profiles = [ l.get('href').split('/')[-1] for l in all_a if isinstance(l.get('href'), str) and l.get('href').find('profile') > 0 and len(l.get_text()) > 2 ] return profiles def charme(self, profiles=[], max_p=10, filename='data/charme.json'): db = {} try: with open(filename, 'r') as in_f: db = json.load(in_f) except: pass visited = 0 for uid in profiles: # Check if profile already in db if uid not in db: if max_p is not None and visited >= max_p: break visited += 1 url = "https://www.adopteunmec.com/profile/" + uid print "Visiting", url page = self._s.get(url) html = BeautifulSoup( page.content.decode('utf-8', 'xmlcharrefreplace'), 'lxml') img_url = html.find(id='img-current-pic')['src'] img_name = img_url.split('/')[-1] date = datetime.datetime.now().strftime("%m-%d %H:%M") db[uid] = { "profile": url, "name": html.find('div', { 'class': 'username' }).get_text(), "img": img_name, "age": html.find('span', { 'class': 'age' }).get_text(), "city": html.find('span', { 'class': 'city' }).get_text(), "desc": html.find(text='Description').find_parent('div').find( 'p').get_text(), "shop": html.find(text='Shopping List').find_parent('div').find( 'p').get_text(), "charmed": date } # Download and save profile pic pic = self._s.get(img_url, stream=True) pic.raw.decode_content = True with open("data/pics/" + img_name, 'wb') as f: shutil.copyfileobj(pic.raw, f) time.sleep(20) # Bit of rest... # Send a charme url = "https://www.adopteunmec.com/events/charm?id=" + uid r = self._s.get(url) if r.json()['member']['id'] != uid: raise 'Something wrong in response' # Write back json json_s = json.dumps( db) # Dump as a string, to write to file and as JS var with open(filename, 'w') as out_f: out_f.write(json_s)