def do_scrape():
"""
Runs the Craigslist scraper, and posts data to slack/sends email. Optionally checks in with Dead Man's Snitch for monitoring.
"""
# Create a slack client.
sc = SlackClient(settings.SLACK_TOKEN)
# Create a mail client and log in
mail = smtplib.SMTP(settings.SMTP_SERVER, settings.SMTP_PORT)
mail.ehlo()
mail.starttls()
mail.login(settings.SMTP_USERNAME, settings.SMTP_PASSWORD)
# Get all the results from craigslist.
all_results = scrape()
print("{}: Got {} results".format(time.ctime(), len(all_results)))
# Post each result to slack and email
for result in all_results:
post_listing_to_slack(sc, result)
send_listing_email(mail, result)
# Quit mail
mail.quit()
# Notify Dead Man's Snitch, if configured
try:
dms_msg = "Found %s results" % (len(all_results))
dms = requests.post(settings.DMS_URL, data={"m": dms_msg})
except Exception:
pass
def do_scrape():
"""
Runs the craigslist scraper, and posts data to slack.
"""
# Create a slack client.
sc = SlackClient(settings.SLACK_TOKEN)
# Get all the results from craigslist.
all_results = []
for area in settings.AREAS:
for query in settings.QUERIES:
all_results += scrape_area(area=area, section='sub', query=query)
print("{}: Got {} results".format(time.ctime(), len(all_results)))
# Post each result to slack.
for result in all_results:
post_listing_to_slack(sc, result, '#sublets')
# Get all the results from craigslist.
all_results = []
for area in settings.AREAS:
for query in settings.QUERIES:
all_results += scrape_area(area=area, section='roo', query=query)
print("{}: Got {} results".format(time.ctime(), len(all_results)))
# Post each result to slack.
for result in all_results:
post_listing_to_slack(sc, result, '#rooms')
def scrape_and_post():
"""
Runs the craigslist scraper, and posts data to slack.
"""
# Create a slack client.
sc = SlackClient(settings.SLACK_TOKEN)
# Get all the results from craigslist.
for section in settings.CRAIGSLIST_HOUSING_SECTION:
all_results = []
for area in settings.AREAS:
results = scrape(
settings.CRAIGSLIST_SITE,
area,
section,
settings.MIN_PRICE[section],
settings.MAX_PRICE[section],
)
all_results += results
print('{}: Got {} results for {}'.format(time.ctime(),
len(all_results),
section))
# Post each result to slack.
channel = settings.SLACK_CHANNEL[section]
for result in all_results:
post_listing_to_slack(sc, channel, result)
def do_scrape():
all_results = []
q = 'concept 2'
for area in settings.AREAS:
for section in settings.SECTIONS:
time.sleep(2)
print("Searching for {} in {} in {}".format(q, section, area))
cl = CraigslistForSale(site=settings.CRAIGSLIST_SITE,
area=area,
category=section,
filters={
'max_price': settings.MAX_PRICE,
'min_price': settings.MIN_PRICE,
'query': q,
'search_titles': 'T'
})
all_results += scrape_area(cl, area, section)
q = 'aether backpack'
for area in settings.AREAS:
for section in settings.SECTIONS:
time.sleep(2)
print("Searching for {} in {} in {}".format(q, section, area))
cl = CraigslistForSale(site=settings.CRAIGSLIST_SITE,
area=area,
category=section,
filters={
'max_price': 170,
'min_price': 50,
'query': q,
'search_titles': 'T'
})
all_results += scrape_area(cl, area, section)
q = 'gregory backpack'
for area in settings.AREAS:
for section in settings.SECTIONS:
time.sleep(2)
print("Searching for {} in {} in {}".format(q, section, area))
cl = CraigslistForSale(site=settings.CRAIGSLIST_SITE,
area=area,
category=section,
filters={
'max_price': 160,
'min_price': 60,
'query': q,
'search_titles': 'T'
})
all_results += scrape_area(cl, area, section)
print("{}: Got {} results".format(time.ctime(), len(all_results)))
sc = SlackClient(settings.SLACK_TOKEN)
for result in all_results:
post_listing_to_slack(sc, result)
def run_scraper():
"""Runs the craigslist scraper and posts data to slack."""
# Create a slack client.
sc = SlackClient(settings.SLACK_TOKEN)
# Get all the results from craigslist.
results = scrape()
for result in results[::-1]: # post the oldest first
post_listing_to_slack(result, sc)
def do_scrape():
"""
Runs the craigslist scraper, and posts data to slack.
"""
# Create a slack client.
sc = SlackClient(settings.SLACK_TOKEN)
# Get all the results from craigslist.
all_results = scrape_area()
print("{}: Got {} results".format(time.ctime(), len(all_results)))
# Post each result to slack.
for result in all_results:
post_listing_to_slack(sc, result)
def scrape_housing():
# Scrape Craigslist for listings.
cl_h = CraigslistHousing(site=settings.CRAIGSLIST_SITE, area='nvn', category=settings.CRAIGSLIST_HOUSING_SECTION,
filters={'min_price': settings.MIN_PRICE_RENT, 'max_price': settings.MAX_PRICE_RENT})
results = []
for result in cl_h.get_results(sort_by='newest', geotagged=True, limit=settings.LIMIT, include_details=True):
results.append(result)
# Filter scraped results for excluded terms.
good_listings = []
x = 0
for result in results:
for term in private.EXCLUDED_TERMS:
if term in result['body'].lower():
break
else:
listing = session.query(Listing).filter_by(
cl_id=result["id"]).first()
# Don't store the listing if it already exists.
if listing is None:
good_listings.append(result)
listing = Listing(
cl_id=result['id'],
link=result['url'],
created=parse(result['datetime']),
name=result['name'],
price=f"${format(float(result['price'][1:]), ',.0f')} CAD",
location=result['where'],
sqft=result['area'],
body=result['body']
)
x += 1
# Save the listing so we don't grab it again.
session.add(listing)
session.commit()
print(f'{time.ctime()}: Found {x} new listings that contained excluded terms.')
# Create slack client.
sc = SlackClient(settings.SLACK_TOKEN)
# Post each result to Slack.
for listing in good_listings:
post_listing_to_slack(sc, listing)
def do_scrape():
"""
Runs the craigslist scraper, and posts data to slack.
"""
# Create a slack client.
sc = SlackClient(settings.SLACK_TOKEN)
# Get all the results from craigslist.
all_results = []
for area in settings.AREAS:
all_results += scrape_area(area)
print("{}: Got {} results".format(time.ctime(), len(all_results)))
# Post each result to slack.
for result in all_results:
post_listing_to_slack(sc, result)
def do_scrape():
"""
runs craigslist scraper, and posts data to slack
"""
#create slack client
sc = SlackClient(settings.SLACK_TOKEN)
#get results from craigslist
all_results = []
for area in settings.AREAS:
all_results += scrape_area(area)
print("{}: Got {} results.".format(time.ctime(), len(all_results)))
#post each result to slack
for result in all_results:
post_listing_to_slack(sc, result)
def do_scrape():
"""
Runs the craigslist scraper, and posts data to slack.
"""
# Create a slack client.
sc = SlackClient(settings.SLACK_TOKEN)
sc.api_call("chat.postMessage", channel=settings.SLACK_CHANNEL,text="WORKING",username='******', icon_emoji=':robot_face:')
# Get all the results from craigslist.
all_results = []
for area in settings.AREAS:
all_results += scrape_area(area)
print("{}: Got {} results".format(time.ctime(), len(all_results)))
# Post each result to slack.
for result in all_results:
post_listing_to_slack(sc, result)
def do_scrape():
"""
Runs the craigslist scraper, and posts data to slack.
"""
# Create a slack client.
sc = SlackClient(settings.SLACK_TOKEN)
# Get all the results from craigslist.
all_results = scrape_area('Bay Area', settings.CRAIGSLIST_SITE1,
settings.CRAIGSLIST_SEARCH1)
all_results += scrape_area('San Diego', settings.CRAIGSLIST_SITE2,
settings.CRAIGSLIST_SEARCH2)
print(len(all_results))
print("{}: Got {} results".format(time.ctime(), len(all_results)))
# Post each result to slack.
for result in all_results:
post_listing_to_slack(sc, result)
def do_scrape():
"""
Runs the craigslist scraper, and posts data to slack.
"""
# Create a slack client.
if not settings.DEV_MODE:
sc = SlackClient(settings.SLACK_TOKEN)
else:
sc = None
# Get all the results from craigslist.
all_results = []
for area in settings.AREAS:
all_results += scrape_area(area)
print("{}: Got {} results".format(time.ctime(), len(all_results)))
# Post each result to slack.
for result in all_results:
post_listing_to_slack(sc, result)
def do_scrape():
"""
Runs the craigslist scraper, and posts data to slack.
"""
# Create a slack client.
sc = SlackClient(settings.SLACK_TOKEN)
# Get all the results from craigslist.
all_results = []
for area in settings.AREAS:
all_results += scrape_area(area)
print("{}: Got {} results".format(time.ctime(), len(all_results)))
# Post each result to slack.
for result in all_results:
try:
post_listing_to_slack(sc, result)
except TypeError:
continue
def do_scrape():
"""
Runs the craigslist scraper, and posts data to slack.
"""
# Create a slack client.
sc = SlackClient(settings.SLACK_TOKEN)
# Get all the results from craigslist.
all_results = []
for area in settings.AREAS:
print("Scraping area {}".format(area))
all_results += scrape_area(area)
print("Scraping area {} completed. Found {} results".format(
area, len(all_results)))
print("{}: Got {} results".format(time.ctime(), len(all_results)))
# Post each result to slack.
for result in all_results:
post_listing_to_slack(sc, result)
def do_scrape():
"""
Runs the craigslist scraper, and posts data to slack.
"""
# Create a slack client.
slack_client = WebClient(settings.SLACK_TOKEN)
# Delete un-reacted old messages if user requested
messages = slack_client.conversations_history(
channel=settings.CHANNEL_ID).get('messages')
if check_for_clear_message(slack_client, messages):
clear_apartments(slack_client, messages)
# Get all the results from craigslist.
all_results = []
for area in settings.AREAS:
all_results += scrape_area(area)
print("{}: Got {} results".format(time.ctime(), len(all_results)))
# Post each result to slack. FIXME: move this below delete thingy
for result in all_results:
post_listing_to_slack(slack_client, result)
