def handler(event, context):
class job:
def __init__(self, title, summary):
self.title = title
self.summary = summary
search_string = event['queryStringParameters']['search']
location = event['queryStringParameters']['state']
client = ScraperAPIClient(os.environ.get('API_KEY'))
monster_url = f'https://www.monster.com/jobs/search?q={search_string}&where={location}&page=1'
page = client.get(url=monster_url, render=True)
soup = BeautifulSoup(page.content, 'html.parser')
results = soup.find(class_='results-page')
job_elems = results.find_all('div', class_='results-card')
jobs = {
"jobs": []
}
for job_elem in job_elems:
title_elem = job_elem.find('div', class_='title-company-location')
summary_elem = job_elem.find('div', class_='results-card-description')
jobs['jobs'].append( job(title_elem.text.strip(), summary_elem.text.strip()))
result = json.dumps(jobs, default = lambda x: x.__dict__)
# result = jobs
return {
'statusCode': 200,
'headers': {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Headers': '*',
'Access-Control-Allow-Methods': 'OPTIONS,POST,GET'
},
'body': result
}
def start_requests(self):
secret = self.properties.properties['Scraper_secret']
print("Properties:", self.properties.properties)
scrape_url = "Scraper_Vomar_Scrape_url"
client = ScraperAPIClient(secret)
for i in self.properties.properties.keys():
if str(i).startswith(scrape_url):
url = self.properties.properties[i]
property_values = self.properties.properties[i].split(";")
priority = i.split("_")[4]
try:
category = property_values[1]
except IndexError as e:
category = property_values[0].split("/")[-1].replace(
"?sort=relevancy+asc", "")
print("Category:", category)
print("Calling yield function for url: ", url)
yield scrapy.Request(client.scrapyGet(url=url),
self.parse,
dont_filter=True,
priority=int(priority),
meta={'category': category})
def start_requests(self):
secret = self.properties.properties['Scraper_secret']
scrape_url = "Scraper_Deen_scrape_url_1"
client = ScraperAPIClient(secret)
url = self.properties.properties[scrape_url] + '?items=6000'
print("URl is:", url)
yield scrapy.Request(client.scrapyGet(url=url),
self.parse,
dont_filter=True)
def __init__(self, key: str, lang="en", period="", ua=""):
assert key != ""
self.client = ScraperAPIClient(key)
self.user_agent = ua
self.__texts = []
self.__titles = []
self.__links = []
self.__results = []
self.__lang = lang
self.__period = period
self.__exec_time = 0
def get_proxies_from_scraper_api(cls, proxy_count=10):
client = ScraperAPIClient('3af7d62e85b75e0271d32f245107a240')
proxies = set()
for i in range(1, proxy_count):
result = client.get(url='http://httpbin.org/ip').text
json_data = json.loads(result)
print(json_data)
proxies.add(json_data["origin"])
print(proxies)
return proxies
def __init__(self, settings):
if not settings.getbool('SCRAPERAPI_ENABLED', True):
raise NotConfigured
self.SCRAPERAPI_KEY = settings.get('SCRAPERAPI_KEY', '')
self.SCRAPERAPI_RENDER = settings.get('SCRAPERAPI_RENDER', False)
self.SCRAPERAPI_PREMIUM = settings.get('SCRAPERAPI_PREMIUM', False)
self.SCRAPERAPI_COUNTRY_CODE = settings.get('SCRAPERAPI_COUNTRY_CODE',
'')
self.SCRAPERAPI_CLIENT = None
try:
self.SCRAPERAPI_CLIENT = ScraperAPIClient(self.SCRAPERAPI_KEY)
except:
raise NotConfigured
def init_scraper(cls, api_key):
""" inits the scraper client
"""
# create the ScraperAPIClient
cls.scraper = ScraperAPIClient(api_key)
def check_setup():
"""Prints to the user if their setup is successful.
"""
from eBayScraper.data_files.api_keys import api_keys
from scraper_api import ScraperAPIClient
all_good = True
if api_keys == [] or d == {}:
all_good = False
#validate api keys
for key in api_keys:
try:
ScraperAPIClient(key).account()
except:
all_good = False
break
os.system('color')
if all_good:
print(colored("Setup is successful!", "green"))
else:
print(
colored(
"Make sure the list in data_files/api_keys.py and the dictionary in data_files/queries.py are not empty!",
"red"))
class ScraperApiProxyMiddleware(object):
def __init__(self, settings):
if not settings.getbool('SCRAPERAPI_ENABLED', True):
raise NotConfigured
self.SCRAPERAPI_KEY = settings.get('SCRAPERAPI_KEY', '')
self.SCRAPERAPI_RENDER = settings.get('SCRAPERAPI_RENDER', False)
self.SCRAPERAPI_PREMIUM = settings.get('SCRAPERAPI_PREMIUM', False)
self.SCRAPERAPI_COUNTRY_CODE = settings.get('SCRAPERAPI_COUNTRY_CODE',
'')
self.SCRAPERAPI_CLIENT = None
try:
self.SCRAPERAPI_CLIENT = ScraperAPIClient(self.SCRAPERAPI_KEY)
except:
raise NotConfigured
@classmethod
def from_crawler(cls, crawler):
o = cls(crawler.settings)
return o
def process_request(self, request, spider):
if 'api.scraperapi.com' not in request.url:
log.info("Process request...")
new_url = self.SCRAPERAPI_CLIENT.scrapyGet(
url=request.url,
render=self.SCRAPERAPI_RENDER,
country_code=self.SCRAPERAPI_COUNTRY_CODE,
premium=self.SCRAPERAPI_PREMIUM)
log.info("New url: {}".format(new_url))
return request.replace(url=new_url)
return
def initialize_client():
"""Initializes the Client's data before starting up the scraping.
"""
Client.data = [(key, ScraperAPIClient(key).account()["requestCount"])
for key in api_keys]
Client.current_client, Client.requests = Client.current_client_and_requests(
)
Client.requests_limit = 1000
def check_availability(URL, scraper_api_key):
# To prevent from being blocked.
sleep(randint(30, 40))
client = ScraperAPIClient(scraper_api_key)
result = client.get(URL)
parsed_html = BeautifulSoup(result.content, 'html.parser')
availability = parsed_html.select("#availability")
if availability:
availability = availability[0].getText().strip().splitlines()[0]
if availability != "Currently unavailable.":
return True
return False
def main():
parser = argparse.ArgumentParser(description='Parses command line arguments')
parser.add_argument('--scraper_api_key', type=str, required=True)
args = parser.parse_args()
client = ScraperAPIClient(args.scraper_api_key)
result = json.loads(client.get(url='http://httpbin.org/ip').text)
print('Rotated proxy IP address = ' + result['origin'])
urls = [
client.scrapyGet(url='http://quotes.toscrape.com/page/1/'),
client.scrapyGet(url='http://quotes.toscrape.com/page/2/'),
]
for url in urls:
r = requests.get(url)
# add parsing logic here
print(r.status_code)
def start_requests(self):
secret = self.properties.properties['Scraper_secret']
print("Properties:", self.properties.properties)
scrape_url = "Scraper_Dirk_Scrape_url"
client = ScraperAPIClient(secret)
for i in self.properties.properties.keys():
if str(i).startswith(scrape_url):
url = self.properties.properties[i]
property_value = self.properties.properties[i].split("/")
priority = i.split("_")[4]
category = property_value[4]
print("Calling yield function for url: ", url)
yield scrapy.Request(client.scrapyGet(url=url),
self.parse,
dont_filter=True,
priority=int(priority),
meta={'category': category})
def __init__(self, api_key):
self._api_key = api_key
self._client = ScraperAPIClient(api_key)
assert api_key is not None
super(ScraperAPI, self).__init__()
self._TIMEOUT = 60
self._session = self._client
self._session.proxies = {}
def start_requests(self):
secret = self.properties.properties['Scraper_secret']
print("Properties:", self.properties.properties)
scrape_url = "Scraper_AH_scrape_url"
client = ScraperAPIClient(secret)
start_urls = list()
priorities = list()
for i in self.properties.properties.keys():
if str(i).startswith(scrape_url):
start_urls.append(self.properties.properties[i])
priorities.append(i.split("_")[4])
print("Key start urls are:", start_urls, "Priorities:", priorities)
urls = set()
for priority, url in zip(priorities, start_urls):
urls.add(url + '?page=' + str(100))
print("Calling yield function for url: ", url + '?page=' + str(100))
yield scrapy.Request(
client.scrapyGet(url=url + '?page=' + str(100)),
self.parse, dont_filter=True, priority=int(priority))
def getInfoInnvictus(self, apiKey='aba275ef5f8f713e086a1a0ab240dd5c'):
""" Method to retrieve a dict given launching shoes on innvictus.com/lanzamientos """
self.client = ScraperAPIClient(apiKey)
result = self.client.get(url='https://www.innvictus.com/lanzamientos')
assert result.status_code == 200, f"Status code {result.status_code}"
soup = BeautifulSoup(result.content, 'html.parser')
links = soup.find('body').find_all("script",
attrs={'type': 'text/javascript'})
s = links[3] # links[3] element gives var products
productVar = re.search(r'\'(.*?)\'', str(s))
productDict = json.loads(
productVar.group(0).replace("'{\"id",
"[{\"id").replace("\"}'", "\"}]"))
#print(productVar.group(0).replace("'{\"id","[{\"id").replace("\"}'","\"}]"))
for tenis in productDict:
tenis['url'] = "https://www.innvictus.com/p/{id}".format(
id=tenis['id'])
self.products = productDict
def product_scraper(url):
from scraper_api import ScraperAPIClient
from bs4 import BeautifulSoup
import json
client = ScraperAPIClient('9aa1dbc863b8334850efccb9be3552f8')
try:
page = client.get(url=url, render=True)
except:
result = {'status_code': 500, 'status': 'scraper api fatal error',
'elapsed_time': '', 'price': '', 'title': ''}
return json.dumps(result)
if page.status_code != 200:
result = {'status_code': page.status_code, 'status': 'error',
'elapsed_time': int(page.elapsed.total_seconds()), 'price': '', 'title': ''}
return json.dumps(result)
else:
soup = BeautifulSoup(page.content, 'html.parser')
result_title = soup.findAll(
"span", {"class": "pdp-mod-product-badge-title"})
result_price = soup.findAll("div", {"class": "pdp-product-price"})
if result_title and result_price:
title = result_title[0].text
price = float(result_price[0].find_next(
'span').text.strip('RM').replace(',', ''))
result = {'status_code': page.status_code, 'status': 'success', 'elapsed_time': int(
page.elapsed.total_seconds()), 'title': title, 'price': price}
return json.dumps(result)
else:
result = {'status_code': 500, 'status': 'blocked/nocontent',
'elapsed_time': int(page.elapsed.total_seconds()), 'title': '', 'price': ''}
return json.dumps(result)
def store_product_info(product_keys):
for key in product_keys:
url = gen_url(base_url, count, offset, page, store_id, key)
try:
client = ScraperAPIClient(settings.SCRAPER_API_KEY)
result = client.get(url=url)
if result.status_code == 200:
res_json = result.json()
elif result.status_code == 500:
raise Exception("Request not successful, status: 500")
elif result.status_code == 403:
raise Exception("Plan max request exceeded, status: 403")
elif result.status_code == 404:
raise Exception("Request not found, status: 404")
elif result.status_code == 410:
raise Exception("Request gone or deleted, status: 410")
except Exception as e:
logger.error("failed to fetch product info, error: " + str(e))
try:
store_to_db(res_json["products"])
except Exception as e:
logger.error("failed to save product info, error: " + str(e))
async def graph(request):
scraper_api_key = getenv("SCRAPER_API_KEY") if getenv("SCRAPER_API_KEY") else None
# Example url where pair graph can be fetched
# url = 'https://gov.capital/forex/usd-eur/'
pair = 'usd-eur'
url = 'https://gov.capital/forex/{}/'.format(pair)
client = ScraperAPIClient(scraper_api_key)
request_result = client.get(url, render=True).text
soup = BeautifulSoup(request_result)
# Removing all divs containing ads
for ads in soup.find_all("div", {"class": "code-block code-block-2"}):
# Removes all ads in fetched page source
ads.decompose()
cleaned_graph_html = '<script src="https://code.jquery.com/jquery-3.5.1.slim.min.js" integrity="sha256-4+XzXVhsDmqanXGHaHvgh1gMQKX40OUvDEBTu8JcmNs=" crossorigin="anonymous"></script>\n\
<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.4.0/Chart.min.js"></script>'
cleaned_graph_html = cleaned_graph_html + soup.find('canvas').next.__str__()
return html(cleaned_graph_html)
def handler(event, context):
client = ScraperAPIClient(os.environ.get('API_KEY'))
URL = "https://www.indeed.com/jobs?q=Entry+Level+Software+Engineer&l=Remote"
page = client.get(url=URL)
print(page)
soup = BeautifulSoup(page.content, 'html.parser')
print(soup)
results = soup.find(id='resultsCol')
print(results)
job_elems = results.find_all('div', class_='jobsearch-SerpJobCard')
titles = []
companies = []
summaries = []
for job_elem in job_elems:
title_elem = job_elem.find('h2', class_='title')
company_elem = job_elem.find('div', class_='sjcl')
summary_elem = job_elem.find('div', class_='summary')
titles.append(title_elem.text.strip())
companies.append(company_elem.text.strip())
summaries.append(summary_elem.text.strip())
response = {
"titles": titles,
"companies": companies,
"summaries": summaries
}
return {
'statusCode': 200,
'headers': {
'Access-Control-Allow-Headers': '*',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'OPTIONS,POST,GET'
},
'body': json.dumps(response)
}
def handler(event, context):
class job:
def __init__(self, title, company, summary):
self.title = title
self.company = company
self.summary = summary
search_string = event['queryStringParameters']['search']
location = event['queryStringParameters']['state']
client = ScraperAPIClient(os.environ.get('API_KEY'))
URL = f"https://www.indeed.com/jobs?q={search_string}&l={location}"
page = client.get(url=URL)
soup = BeautifulSoup(page.content, 'html.parser')
results = soup.find(id='resultsCol')
job_elems = results.find_all('div', class_='jobsearch-SerpJobCard')
jobs = {"jobs": []}
for job_elem in job_elems:
title_elem = job_elem.find('h2', class_='title')
company_elem = job_elem.find('div', class_='sjcl')
summary_elem = job_elem.find('div', class_='summary')
jobs['jobs'].append(
job(title_elem.text.strip(), company_elem.text.strip(),
summary_elem.text.strip()))
result = json.dumps(jobs, default=lambda x: x.__dict__)
return {
'statusCode': 200,
'headers': {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Headers': '*',
'Access-Control-Allow-Methods': 'OPTIONS,POST,GET'
},
'body': result.replace("\\n", " ")
}
def start_requests(self):
secret = self.properties.properties['Scraper_secret']
print("Properties:", self.properties.properties)
scrape_url = "Scraper_Coop_scrape_url"
client = ScraperAPIClient(secret)
start_urls = list()
priorities = list()
categories = list()
for i in self.properties.properties.keys():
if str(i).startswith(scrape_url):
property_value = self.properties.properties[i].split(";")
start_urls.append(property_value[0])
priorities.append(i.split("_")[4])
categories.append(property_value[1])
# print("Key start urls are:", start_urls, "Priorities:", priorities)
for priority, url, category in zip(priorities, start_urls, categories):
print("Calling yield function for url: ", url)
yield scrapy.Request(client.scrapyGet(url=url),
self.parse,
dont_filter=True,
priority=int(priority),
meta={'category': category})
def handler(event, context):
client = ScraperAPIClient(os.environ.get('API_KEY'))
monster_url = "https://www.monster.com/jobs/search?q=Software+Engineer&where=Nashville%2C+TN&page=6"
page = client.get(url=monster_url, render=True)
soup = BeautifulSoup(page.content, 'html.parser')
results = soup.find(class_='results-page')
job_elems = results.find_all('div', class_='results-card')
titles = []
summaries = []
for job_elem in job_elems:
title_elem = job_elem.find('div', class_='title-company-location')
summary_elem = job_elem.find('div', class_='results-card-description')
titles.append(title_elem.text.strip())
summaries.append(summary_elem.text.strip())
response = {"titles": titles, "summaries": summaries}
return {
'statusCode': 200,
'headers': {
'Access-Control-Allow-Headers': '*',
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'OPTIONS,POST,GET'
},
'body': json.dumps(response)
}
class ScraperApi(Base):
def __init__(self, key: str, max_retry):
self.max_retry = max_retry
self.client = ScraperAPIClient(key)
def get(self, url: str, headers: Dict[str, str]):
if not headers:
headers = {}
if "User-Agent" not in headers.keys():
headers["User-Agent"] = USER_AGENTS[randint(
0,
len(USER_AGENTS) - 1)]
return self.client.get(url, headers, retry=self.max_retry).text
def get_json(self, url: str, headers: Union[Dict[str, str], None]):
return json.loads(self.get(url, headers))
class GoogleNews:
def __init__(self, key: str, lang="en", period="", ua=""):
assert key != ""
self.client = ScraperAPIClient(key)
self.user_agent = ua
self.__texts = []
self.__titles = []
self.__links = []
self.__results = []
self.__lang = lang
self.__period = period
self.__exec_time = 0
def set_lang(self, lang):
self.__lang = lang
def search(self, q: Union[List[str], str], p: Union[List[int], int],
start: datetime, end: datetime) -> List[dict]:
"""
Searches for a term in google news and retrieves the first page into __results.
Parameters:
key = the search term
"""
start_time = time()
if isinstance(q, str):
q = [q]
if isinstance(p, int):
p = [p]
elif len(p) < 1:
p = [1]
for query in q:
for page in p:
out = self.scrape_page(query, page, start, end)
for o in out:
if o["title"] not in self.__titles:
self.__results.append(o)
self.__links.append(o["link"])
self.__texts.append(o["title"] + " " + o["desc"])
self.__exec_time = time() - start_time
return self.__results
def scrape_page(self,
q: str,
page: int,
start: datetime,
end: datetime,
attempts=0):
"""
page = number of the page to be retrieved
"""
payload = {
'q': q,
'lr': f'lang_{self.__lang}',
'tbs': f"lr:lang_1{self.__lang}",
'tbm': 'nws',
'start': (10 * (page - 1)),
}
out: List[dict] = []
if start is not None and end is not None:
payload['tbs'] += f",cdr:1,cd_min:{start.strftime(DATE_FMT)}," \
f"cd_max:{end.strftime(DATE_FMT)}"
try:
page = self.client.get(url=GOOG_URL + "?" +
parse.urlencode(payload)).text
content = Soup(page, "html.parser")
except Exception as e:
attempts += 1
if attempts > 5:
print(f"ERROR TRYING TO LOAD CONTENT: {e}")
raise e
sleep(0.1 * attempts)
self.scrape_page(q, page, start, end, attempts)
try:
result = content.find_all("div", id="search")[0].find_all("g-card")
except IndexError:
# no results were found
return out
for item in result:
try:
out.append({
"title":
item.find("div", {
"role": "heading"
}).text.replace("\n", ""),
"link":
item.find("a").get("href"),
"media":
item.findAll("g-img")[1].parent.text,
"date":
item.find("div", {
"role": "heading"
}).next_sibling.findNext('div').findNext('div').text,
"desc":
item.find("div", {
"role": "heading"
}).next_sibling.findNext('div').text.replace("\n", ""),
"image":
item.findAll("g-img")[0].find("img").get("src")
})
except Exception:
pass
return out
def get_results(self) -> List[dict]:
"""Returns the __results."""
return self.__results
def get_text(self) -> List[str]:
"""Returns only the __texts of the __results."""
return self.__texts
def get_links(self) -> List[str]:
"""Returns only the __links of the __results."""
return self.__links
def clear(self):
self.__texts = []
self.__links = []
self.__results = []
self.__titles = []
self.__exec_time = 0
import scrapy
from http.cookies import SimpleCookie
from ..items import WgUpdateItem
from scrapy.loader import ItemLoader
import re
from scraper_api import ScraperAPIClient
from scraperapikey import key
client = ScraperAPIClient(key)
class WgSpider(scrapy.Spider):
name = 'wg'
cookies1 = {}
ot, date_from, date_to, fur, city_id, start_date_value, end_date_value, api = '', '', '', '', '', '', '', ''
def start_requests(self):
rawdata = f" last_city={self.city_id}; last_cat=0; last_type=0"
cookie = SimpleCookie()
cookie.load(rawdata)
for key, morsel in cookie.items():
self.cookies1[key] = morsel.value
url = f"https://www.wg-gesucht.de/en/wg-zimmer-in-cityname.{self.city_id}.0.1.0.html?offer_filter=1&city_id={self.city_id}&noDeact=1&dFr={self.date_from}&dTo={self.date_to}&categories%5B%5D=0&rent_types%5B%5D=0&ot%5B%5D={self.ot}"
if (self.api == '1'):
yield scrapy.Request(client.scrapyGet(url=url),
cookies=self.cookies1,
callback=self.parse)
else:
yield scrapy.Request(url=url,
from scrapy import Request
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy_selenium import SeleniumRequest
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
# from scrapy.utils.response import open_in_browser
from scraper_api import ScraperAPIClient
from ..items import CompanyItem
from dotenv import load_dotenv
load_dotenv()
client = ScraperAPIClient(os.environ["SCRAPER_API"])
class BbbSpiderSpider(CrawlSpider):
name = "bbb_spider"
def start_requests(self):
with open("zip_cat_list3.csv") as file:
dreader = DictReader(file)
zip_cat = list(dreader)
for req in zip_cat:
print("-" * 40)
print(req["zip_code"])
# count URLs in the scheduler
import proxies
import requests
from proxies import random_proxy
from scraper_api import ScraperAPIClient
import random
def random_line(fname):
lines = open(fname).read().splitlines()
return random.choice(lines)
client = ScraperAPIClient('d0224166b175ddca1f18dd5b5cca66a5')
result = client.get(url = 'http://httpbin.org/headers',
headers={
'useragent':random_line('user-agents.txt') }
)
print(result.text);
class snkrs:
def __init__(self) -> None:
self.products = {}
def getInfoInnvictus(self, apiKey='aba275ef5f8f713e086a1a0ab240dd5c'):
""" Method to retrieve a dict given launching shoes on innvictus.com/lanzamientos """
self.client = ScraperAPIClient(apiKey)
result = self.client.get(url='https://www.innvictus.com/lanzamientos')
assert result.status_code == 200, f"Status code {result.status_code}"
soup = BeautifulSoup(result.content, 'html.parser')
links = soup.find('body').find_all("script",
attrs={'type': 'text/javascript'})
s = links[3] # links[3] element gives var products
productVar = re.search(r'\'(.*?)\'', str(s))
productDict = json.loads(
productVar.group(0).replace("'{\"id",
"[{\"id").replace("\"}'", "\"}]"))
#print(productVar.group(0).replace("'{\"id","[{\"id").replace("\"}'","\"}]"))
for tenis in productDict:
tenis['url'] = "https://www.innvictus.com/p/{id}".format(
id=tenis['id'])
self.products = productDict
def selectSaveTargetShoes(self, save=False):
i = 0
for shoes in self.products:
print(f"Selelction {i}")
print("Shoe: ", shoes['name2'])
print("Launching date: ", shoes['realdate'])
print("Price: ", shoes['price'], "\n")
i += 1
selectedShoe = [
int(x) for x in input("Select shoe number to track\n").split()
]
#print(selectedShoe)
def takeFromDict(dct, listElements):
return [dct[element] for element in listElements]
self.products = takeFromDict(self.products, selectedShoe)
if save == True:
with open('TargetShoes.txt', 'w') as f:
for shoe in self.products:
f.write(shoe['url'])
f.write("\n")
def isAvailable(self, fileName=None):
def sendDiscordMessage(self, message='Hello world'):
from discord import Webhook, RequestsWebhookAdapter
webhook = Webhook.from_url(
"https://discord.com/api/webhooks/811294684386426890/3SY7GtmBAwyjDM6qr73eDRqrzjRJZ2u0vlShoyOvvf_cCUNvv6YJqiGPI1udVWWqipVp",
adapter=RequestsWebhookAdapter())
webhook.send(message)
if fileName == None:
for url in self.products:
notAvailable = self.client.get(url['url'])
assert notAvailable.status_code == 200, f"Status code {notAvailable.status_code}"
soupNotAvailable = BeautifulSoup(notAvailable.content,
'html.parser')
try:
notFoundClass = soupNotAvailable.find('body').find(
"div", attrs={
'class': 'pdp-notFound'
}).text.strip()
notFoundTitle = soupNotAvailable.find('title').text.strip()
except:
notFoundClass = ''
notFoundTitle = ''
if "No encontrado" in notFoundTitle or "Este producto no" in notFoundClass:
url['Available'] = False
message = url['name2'], " no disponible aun"
print(message)
return False
else:
message = url['name2'], " disponible", url['url']
print(message)
url['Available'] = True
sendDiscordMessage(self, message=message)
return True
else:
with open(fileName, 'r') as targetShoes:
urls = [url.strip() for url in targetShoes]
for url in urls:
notAvailable = self.client.get(url, headers=self.headers)
soupNotAvailable = BeautifulSoup(notAvailable.content,
'html.parser')
try:
notFoundClass = soupNotAvailable.find('body').find(
"div", attrs={
'class': 'pdp-notFound'
}).text.strip()
notFoundTitle = soupNotAvailable.find(
'title').text.strip()
except:
notFoundClass = ''
notFoundTitle = ''
if "No encontrado" in notFoundTitle or "Este producto no" in notFoundClass:
print(url, " no disponible aun\n")
else:
print(url, " disponible \n")
class TruliaSpider(scrapy.Spider):
client = ScraperAPIClient('22c786c81d0f6a84eb1312a0d6c6aec5')
name = 'trulia'
allowed_domains = ['trulia.com']
custom_settings = {'FEED_URI': '/tmp/data/data_for_rent_%(time)s.jl',
'FEED_FORMAT': 'jsonlines'}
def __init__(self, state='CA', cities=['Oakland','Alameda','Berkeley','Emeryville'], *args, **kwargs):
super().__init__(*args, **kwargs)
self.state = state
self.cities = cities
url = 'https://trulia.com/for_rent/{city},{state}/2000-4000_price/1300p_sqft/'
self.base_urls = [url.format(state=state, city=city) for city in self.cities]
self.start_urls = [self.client.scrapyGet(url=url) for url in self.base_urls]
self.le = LinkExtractor(restrict_xpaths='//*[@id="resultsColumn"]/div[1]')
def parse(self, response):
N = self.get_number_of_pages_to_scrape(response)
self.logger.info("Determined that property pages are contained on {N} different index pages, each containing at most 30 properties. Proceeding to scrape each index page...".format(N=N))
self.logger.info("Response URL is {}".format(response.url))
for base_url in self.base_urls:
for url in [base_url + "{n}_p/".format(n=n) for n in range(1, N+1)]:
yield scrapy.Request(url=self.client.scrapyGet(url), callback=self.parse_index_page, dont_filter=True)
@staticmethod
def get_number_of_pages_to_scrape(response):
pagination = response.xpath('//*[@data-testid="pagination-caption"]/text()')
if pagination.re(r'1-30 of ([\d,]+) Results'):
number_of_results = int(pagination.re(r'1-30 of ([\d,]+) Results')[0])
return math.ceil(number_of_results/30)
else:
return 1
def parse_index_page(self, response):
links = self.le.extract_links(response)
for link in links:
link.url = link.url.replace('https://api.scraperapi.com','https://trulia.com')
yield scrapy.Request(url=self.client.scrapyGet(link.url), callback=self.parse_property_page, dont_filter=True)
def parse_property_page(self, response):
l = TruliaItemLoader(item=TruliaItem(), response=response)
self.load_common_fields(item_loader=l, response=response)
item = l.load_item()
self.post_process(item=item)
return item
@staticmethod
def load_common_fields(item_loader, response):
'''Load field values which are common to "on sale" and "recently sold" properties.'''
item_loader.add_value('url', urllib.parse.unquote(response.url[32:-60]))
item_loader.add_xpath('address', '//*[@data-testid="home-details-summary-headline"]/text()')
item_loader.add_xpath('city_state', '//*[@data-testid="home-details-summary-city-state"]/text()')
item_loader.add_xpath('price', '//*[@data-testid="on-market-price-details"]/div/text()', re=r'\$([\d,]+)')
item_loader.add_xpath('neighborhood', '//*[@data-testid="neighborhood-link"]/text()')
item_loader.add_xpath('neighborhood_url', '//*[@data-testid="neighborhood-link"]/@href')
fact_list = item_loader.nested_xpath('//*[@data-testid="facts-list"]')
fact_list.add_xpath('bedrooms', xpath='.//*[@data-testid="bed"]/div/div[2]/text()', re=r'(\d+) (?:Beds|Bed|beds|bed)$')
fact_list.add_xpath('bathrooms', xpath='.//*[@data-testid="bath"]/div/div[2]/text()', re=r'(\d+\.?[0-9]*) (?:Baths|Bath|baths|bath)$')
fact_list.add_xpath('area', xpath='.//*[@data-testid="floor"]/div/div[2]/text()', re=r'([\d, -]+)')
item_loader.add_xpath('telephone', '//*[@data-testid="home-description-text-description-phone-number"]/div/div[2]/text()')
item_loader.add_xpath('description', '//*[@data-testid="home-description-text-description-text"]/text()')
item_loader.add_xpath('tags','//*[@data-testid="hero-image-property-tag-1"]/span/text()')
item_loader.add_xpath('tags','//*[@data-testid="hero-image-property-tag-2"]/span/text()')
item_loader.add_xpath('tags','//*[@data-testid="hero-image-property-tag-3"]/span/text()')
item_loader.add_xpath('tags','//*[@data-testid="hero-image-property-tag-4"]/span/text()')
item_loader.add_xpath('tags','//*[@data-testid="hero-image-property-tag-5"]/span/text()')
features = item_loader.nested_xpath('//*[@data-testid="structured-amenities-table-category"]')
#features without a sub-header are 1 div below /table/tbody/tr/td
features.add_xpath('features', xpath='.//table/tbody/tr/td/div/li/text()')
#features with a sub-header are 2 divs below /table/tbody/tr/td
#should we try to get the sub-header and associate it to the value? it would be in .//table/tbody/tr/td/div/div/text()
features.add_xpath('features', xpath='.//table/tbody/tr/td/div[2]/li/text()')
item_loader.add_xpath('attribute_values', '//*[@data-testid="wls-attribute"]/div/div/div/div/div/div[2]/div/text()', re=r'\d+')
item_loader.add_xpath('attribute_names', '//*[@data-testid="wls-attribute"]/div/div/div[2]/div/div[2]/div/text()')
@staticmethod
def post_process(item):
'''Add any additional data to an item after loading it'''
s8 = ['section 8', 'section8', 'GoSection8.com']
np = ['no pet', 'no pets', 'pets not allowed', 'no pets allowed','pets are not allowed']
p = ['dog', 'cat', 'cats','small dogs', 'pet considered','pets considered','pets allowed','pets ok','pets okay','pets negotiable']
item['features'] = list(dict.fromkeys(item.get('features',[]))) #remove list duplicates
features = [str.casefold(f) for f in item.get('features',[])]
description = [str.casefold(d) for d in item.get('description',[])]
f_d = features + description
tags = item.get('tags',[])
section8 = [item for s8_phrase in s8 for item in f_d if s8_phrase in item]
item['section8'] = False if not section8 else True
if 'INCOME RESTRICTED' in tags or 'SENIOR HOUSING' in tags:
item['section8'] = True
no_pets = [item for np_phrase in np for item in f_d if np_phrase in item]
yes_pets = [item for p_phrase in p for item in f_d if p_phrase in item]
if no_pets:
item['pets'] = False
elif yes_pets:
item['pets'] = True
else:
item['pets'] = None
if 'PET FRIENDLY' in tags:
item['pets'] = True
if 'FURNISHED' in tags:
item['furnished'] = True
deposit = ' '.join([i for i in f_d if 'deposit' in i])
deposit = [i for i in deposit.split() if i.replace('$','').replace(',','').isdigit()]
if deposit:
item['deposit'] = int(deposit[0].replace('$','').replace(',',''))
year_built = ' '.join([i for i in f_d if 'year built' in i])
year_built = [i for i in year_built.split() if i.isdigit()]
if year_built:
item['year_built'] = int(year_built[0])
property_type = ' '.join([i for i in f_d if 'property type' in i])
if property_type:
item['property_type'] = property_type.replace('property type: ','')
days_on_market = ' '.join([i for i in f_d if 'days on market' in i])
days_on_market = [i for i in days_on_market.split() if i.isdigit()]
if days_on_market:
item['days_on_market'] = int(days_on_market[0])
parking = ' '.join([i for i in f_d if 'parking' in i])
if parking:
item['parking'] = parking.replace('parking: ','')
floors = ' '.join([i for i in f_d if 'floors' in i])
if floors:
item['floors'] = floors.replace('floors: ','')
heating = ' '.join([i for i in f_d if 'heating' in i])
if heating:
item['heating'] = heating.replace('heating: ','')
ac = [i for i in f_d if 'air conditioning' in i]
if ac:
item['ac'] = True
fitness = [i for i in f_d if 'fitness center' in i]
if fitness:
item['fitness'] = True
if 'attribute_names' in item and 'attribute_values' in item:
item['attributes'] = {}
for idx, val in enumerate(item['attribute_names']):
item['attributes'][val] = item['attribute_values'][idx]
del item['attribute_names']
del item['attribute_values']