def scrape_shoes(key): url = URLS.get(key) response = (requests.get(url, timeout=5)) soup = BeautifulSoup(response.content, "html.parser") table = soup.find_all("table", {"class": "roundy"}) items = {} for tableNumber in range(2, 8): for tr in table[tableNumber].find_all("tr")[2:]: name = tr.find_all("td")[0].text.strip() item = { "name": name, # "imageLink": tr.find_all("td")[1].find_all("a")[0]["href"], "priceBuy": parse_price(tr.find_all("td")[2].text), "priceSell": parse_price(tr.find_all("td")[3].text), "source": parse_source(tr.find_all("td")[4]), "variations": parse_variations(tr.find_all("td")[5]), "variationImageLinks": get_image_links(tr.find_all("td")[5].find_all("img")) } if tr.find_all("td")[1].find_all("a"): item["imageLink"] = tr.find_all("td")[1].find_all( "a")[0]["href"] items[name] = item dump_data(items, "clothing/" + key) return items
def scrape_furniture_housewares(key): url = URLS.get(key) response = (requests.get(url, timeout=5)) soup = BeautifulSoup(response.content, "html.parser") table = soup.find_all("table", {"class": "roundy"}) items = {} print(table[3]("tr")) for tr in table[3]("tr")[2:]: name = tr.find_all("td")[1].text.strip() item = { "name": name, # "imageLink": tr.find_all("td")[1].find_all("a")[0]["href"], "priceBuy": parse_price(tr.find_all("td")[2].text), "priceSell": parse_price(tr.find_all("td")[3].text), "source": parse_source(tr.find_all("td")[4]), "variations": parse_variations(tr.find_all("td")[5]), "customization": False, "sizeLink": tr.find_all("td")[6].img.get("data-src") } if tr.find_all("td")[1].find_all("a"): item["imageLink"] = tr.find_all("td")[0].find_all("a")[0]["href"] items[name] = item dump_data(items, "furniture/" + key) return items