def scrape_shoes(key):
url = URLS.get(key)
response = (requests.get(url, timeout=5))
soup = BeautifulSoup(response.content, "html.parser")
table = soup.find_all("table", {"class": "roundy"})
items = {}
for tableNumber in range(2, 8):
for tr in table[tableNumber].find_all("tr")[2:]:
name = tr.find_all("td")[0].text.strip()
item = {
"name":
name,
# "imageLink": tr.find_all("td")[1].find_all("a")[0]["href"],
"priceBuy":
parse_price(tr.find_all("td")[2].text),
"priceSell":
parse_price(tr.find_all("td")[3].text),
"source":
parse_source(tr.find_all("td")[4]),
"variations":
parse_variations(tr.find_all("td")[5]),
"variationImageLinks":
get_image_links(tr.find_all("td")[5].find_all("img"))
}
if tr.find_all("td")[1].find_all("a"):
item["imageLink"] = tr.find_all("td")[1].find_all(
"a")[0]["href"]
items[name] = item
dump_data(items, "clothing/" + key)
return items
def scrape_wallpapers(key):
url = URLS.get(key)
response = (requests.get(url, timeout=5))
soup = BeautifulSoup(response.content, "html.parser")
table = soup.find_all("table", {"class": "sortable"})
items = {}
for tr in table[0].find_all("tr")[1:]:
name = tr.find_all("td")[0].a.text
item = {
"name": name,
}
if tr.find_all("a")[1]['href']:
item["imageLink"] = tr.find_all("a")[1]['href']
if tr.find_all("td")[2]:
item["materials"] = separate_by_br(
tr.find_all("td")[2]).strip("\n").split(",")
item["materialsImageLink"] = get_image_links(
tr.find_all("td")[2].find_all("img"))
if tr.find_all("td")[3].find_all("a"):
item["sizeLink"] = tr.find_all("td")[3].find_all("a")[0]['href']
if tr.find_all("td")[4].text:
if (tr.find_all("td")[4].text.strip('\n').splitlines() == []):
pass
else:
item["obtainedFrom"] = tr.find_all("td")[4].text.strip(
'\n').splitlines()
if tr.find_all("td")[5].text.strip().replace(",", ""):
item["price"] = int(
tr.find_all("td")[5].text.strip().replace(",", ""))
items[name] = item
dump_data(items, "crafting/" + key)
return items
def scrape_DIYothers(key):
url = URLS.get(key)
response = (requests.get(url, timeout=5))
soup = BeautifulSoup(response.content, "html.parser")
table = soup.find_all("table", {"class": "roundy"})
items = {}
for tr in table[2].find_all("tr")[1:]:
name = tr.find_all("td")[0].a.text
item = {
"name":
name,
"imageLink":
tr.find_all("a")[1]['href'],
"materials":
separate_by_br(
tr.find_all("td")[2]).lstrip().strip("\n").split(","),
"materialsImageLink":
get_image_links(tr.find_all("td")[2].find_all("img")),
"sizeImageLink":
tr.find_all("td")[3].img.get("data-src"),
"obtainedFrom":
tr.find_all("td")[4].text.strip().strip("\n").splitlines(),
"price":
parse_price(tr.find_all("td")[5].text)
}
if (item["obtainedFrom"] == ["Nook Stop (1,000 )"
]): # TODO: rewrite this lazy code
item["obtainedFrom"] = ["Nook Stop (1,000 Nook Miles)"]
items[name] = item
dump_data(items, "crafting/" + key)
return items
def scrape_equipments(key):
url = URLS.get(key)
response = (requests.get(url, timeout=5))
soup = BeautifulSoup(response.content, "html.parser")
table = soup.find_all("table", {"class": "sortable"})
items = {}
for tr in table[0].find_all("tr")[1:]:
name = tr.find_all("td")[0].a.text
item = {
"name":
name,
"imageLink":
tr.find_all("a")[1]['href'],
"materials":
separate_by_br(
tr.find_all("td")[2]).lstrip().strip("\n").split(","),
"materialsImageLink":
get_image_links(tr.find_all("td")[2].find_all("img")),
"sizeImageLink":
tr.find_all("td")[3].img.get("data-src"),
"obtainedFrom":
tr.find_all("td")[4].text.strip().strip("\n").splitlines(),
"price":
parse_price(tr.find_all("td")[5].text)
}
items[name] = item
dump_data(items, "crafting/" + key)
return items