def do_scrape(): """ Runs the Craigslist scraper, and posts data to slack/sends email. Optionally checks in with Dead Man's Snitch for monitoring. """ # Create a slack client. sc = SlackClient(settings.SLACK_TOKEN) # Create a mail client and log in mail = smtplib.SMTP(settings.SMTP_SERVER, settings.SMTP_PORT) mail.ehlo() mail.starttls() mail.login(settings.SMTP_USERNAME, settings.SMTP_PASSWORD) # Get all the results from craigslist. all_results = scrape() print("{}: Got {} results".format(time.ctime(), len(all_results))) # Post each result to slack and email for result in all_results: post_listing_to_slack(sc, result) send_listing_email(mail, result) # Quit mail mail.quit() # Notify Dead Man's Snitch, if configured try: dms_msg = "Found %s results" % (len(all_results)) dms = requests.post(settings.DMS_URL, data={"m": dms_msg}) except Exception: pass
def do_scrape(): """ Runs the craigslist scraper, and posts data to slack. """ # Create a slack client. sc = SlackClient(settings.SLACK_TOKEN) # Get all the results from craigslist. all_results = [] for area in settings.AREAS: for query in settings.QUERIES: all_results += scrape_area(area=area, section='sub', query=query) print("{}: Got {} results".format(time.ctime(), len(all_results))) # Post each result to slack. for result in all_results: post_listing_to_slack(sc, result, '#sublets') # Get all the results from craigslist. all_results = [] for area in settings.AREAS: for query in settings.QUERIES: all_results += scrape_area(area=area, section='roo', query=query) print("{}: Got {} results".format(time.ctime(), len(all_results))) # Post each result to slack. for result in all_results: post_listing_to_slack(sc, result, '#rooms')
def scrape_and_post(): """ Runs the craigslist scraper, and posts data to slack. """ # Create a slack client. sc = SlackClient(settings.SLACK_TOKEN) # Get all the results from craigslist. for section in settings.CRAIGSLIST_HOUSING_SECTION: all_results = [] for area in settings.AREAS: results = scrape( settings.CRAIGSLIST_SITE, area, section, settings.MIN_PRICE[section], settings.MAX_PRICE[section], ) all_results += results print('{}: Got {} results for {}'.format(time.ctime(), len(all_results), section)) # Post each result to slack. channel = settings.SLACK_CHANNEL[section] for result in all_results: post_listing_to_slack(sc, channel, result)
def do_scrape(): all_results = [] q = 'concept 2' for area in settings.AREAS: for section in settings.SECTIONS: time.sleep(2) print("Searching for {} in {} in {}".format(q, section, area)) cl = CraigslistForSale(site=settings.CRAIGSLIST_SITE, area=area, category=section, filters={ 'max_price': settings.MAX_PRICE, 'min_price': settings.MIN_PRICE, 'query': q, 'search_titles': 'T' }) all_results += scrape_area(cl, area, section) q = 'aether backpack' for area in settings.AREAS: for section in settings.SECTIONS: time.sleep(2) print("Searching for {} in {} in {}".format(q, section, area)) cl = CraigslistForSale(site=settings.CRAIGSLIST_SITE, area=area, category=section, filters={ 'max_price': 170, 'min_price': 50, 'query': q, 'search_titles': 'T' }) all_results += scrape_area(cl, area, section) q = 'gregory backpack' for area in settings.AREAS: for section in settings.SECTIONS: time.sleep(2) print("Searching for {} in {} in {}".format(q, section, area)) cl = CraigslistForSale(site=settings.CRAIGSLIST_SITE, area=area, category=section, filters={ 'max_price': 160, 'min_price': 60, 'query': q, 'search_titles': 'T' }) all_results += scrape_area(cl, area, section) print("{}: Got {} results".format(time.ctime(), len(all_results))) sc = SlackClient(settings.SLACK_TOKEN) for result in all_results: post_listing_to_slack(sc, result)
def run_scraper(): """Runs the craigslist scraper and posts data to slack.""" # Create a slack client. sc = SlackClient(settings.SLACK_TOKEN) # Get all the results from craigslist. results = scrape() for result in results[::-1]: # post the oldest first post_listing_to_slack(result, sc)
def do_scrape(): """ Runs the craigslist scraper, and posts data to slack. """ # Create a slack client. sc = SlackClient(settings.SLACK_TOKEN) # Get all the results from craigslist. all_results = scrape_area() print("{}: Got {} results".format(time.ctime(), len(all_results))) # Post each result to slack. for result in all_results: post_listing_to_slack(sc, result)
def scrape_housing(): # Scrape Craigslist for listings. cl_h = CraigslistHousing(site=settings.CRAIGSLIST_SITE, area='nvn', category=settings.CRAIGSLIST_HOUSING_SECTION, filters={'min_price': settings.MIN_PRICE_RENT, 'max_price': settings.MAX_PRICE_RENT}) results = [] for result in cl_h.get_results(sort_by='newest', geotagged=True, limit=settings.LIMIT, include_details=True): results.append(result) # Filter scraped results for excluded terms. good_listings = [] x = 0 for result in results: for term in private.EXCLUDED_TERMS: if term in result['body'].lower(): break else: listing = session.query(Listing).filter_by( cl_id=result["id"]).first() # Don't store the listing if it already exists. if listing is None: good_listings.append(result) listing = Listing( cl_id=result['id'], link=result['url'], created=parse(result['datetime']), name=result['name'], price=f"${format(float(result['price'][1:]), ',.0f')} CAD", location=result['where'], sqft=result['area'], body=result['body'] ) x += 1 # Save the listing so we don't grab it again. session.add(listing) session.commit() print(f'{time.ctime()}: Found {x} new listings that contained excluded terms.') # Create slack client. sc = SlackClient(settings.SLACK_TOKEN) # Post each result to Slack. for listing in good_listings: post_listing_to_slack(sc, listing)
def do_scrape(): """ Runs the craigslist scraper, and posts data to slack. """ # Create a slack client. sc = SlackClient(settings.SLACK_TOKEN) # Get all the results from craigslist. all_results = [] for area in settings.AREAS: all_results += scrape_area(area) print("{}: Got {} results".format(time.ctime(), len(all_results))) # Post each result to slack. for result in all_results: post_listing_to_slack(sc, result)
def do_scrape(): """ runs craigslist scraper, and posts data to slack """ #create slack client sc = SlackClient(settings.SLACK_TOKEN) #get results from craigslist all_results = [] for area in settings.AREAS: all_results += scrape_area(area) print("{}: Got {} results.".format(time.ctime(), len(all_results))) #post each result to slack for result in all_results: post_listing_to_slack(sc, result)
def do_scrape(): """ Runs the craigslist scraper, and posts data to slack. """ # Create a slack client. sc = SlackClient(settings.SLACK_TOKEN) sc.api_call("chat.postMessage", channel=settings.SLACK_CHANNEL,text="WORKING",username='******', icon_emoji=':robot_face:') # Get all the results from craigslist. all_results = [] for area in settings.AREAS: all_results += scrape_area(area) print("{}: Got {} results".format(time.ctime(), len(all_results))) # Post each result to slack. for result in all_results: post_listing_to_slack(sc, result)
def do_scrape(): """ Runs the craigslist scraper, and posts data to slack. """ # Create a slack client. sc = SlackClient(settings.SLACK_TOKEN) # Get all the results from craigslist. all_results = scrape_area('Bay Area', settings.CRAIGSLIST_SITE1, settings.CRAIGSLIST_SEARCH1) all_results += scrape_area('San Diego', settings.CRAIGSLIST_SITE2, settings.CRAIGSLIST_SEARCH2) print(len(all_results)) print("{}: Got {} results".format(time.ctime(), len(all_results))) # Post each result to slack. for result in all_results: post_listing_to_slack(sc, result)
def do_scrape(): """ Runs the craigslist scraper, and posts data to slack. """ # Create a slack client. if not settings.DEV_MODE: sc = SlackClient(settings.SLACK_TOKEN) else: sc = None # Get all the results from craigslist. all_results = [] for area in settings.AREAS: all_results += scrape_area(area) print("{}: Got {} results".format(time.ctime(), len(all_results))) # Post each result to slack. for result in all_results: post_listing_to_slack(sc, result)
def do_scrape(): """ Runs the craigslist scraper, and posts data to slack. """ # Create a slack client. sc = SlackClient(settings.SLACK_TOKEN) # Get all the results from craigslist. all_results = [] for area in settings.AREAS: all_results += scrape_area(area) print("{}: Got {} results".format(time.ctime(), len(all_results))) # Post each result to slack. for result in all_results: try: post_listing_to_slack(sc, result) except TypeError: continue
def do_scrape(): """ Runs the craigslist scraper, and posts data to slack. """ # Create a slack client. sc = SlackClient(settings.SLACK_TOKEN) # Get all the results from craigslist. all_results = [] for area in settings.AREAS: print("Scraping area {}".format(area)) all_results += scrape_area(area) print("Scraping area {} completed. Found {} results".format( area, len(all_results))) print("{}: Got {} results".format(time.ctime(), len(all_results))) # Post each result to slack. for result in all_results: post_listing_to_slack(sc, result)
def do_scrape(): """ Runs the craigslist scraper, and posts data to slack. """ # Create a slack client. slack_client = WebClient(settings.SLACK_TOKEN) # Delete un-reacted old messages if user requested messages = slack_client.conversations_history( channel=settings.CHANNEL_ID).get('messages') if check_for_clear_message(slack_client, messages): clear_apartments(slack_client, messages) # Get all the results from craigslist. all_results = [] for area in settings.AREAS: all_results += scrape_area(area) print("{}: Got {} results".format(time.ctime(), len(all_results))) # Post each result to slack. FIXME: move this below delete thingy for result in all_results: post_listing_to_slack(slack_client, result)