def parse(self) -> List[ProductListing]:
""" Parse the product listing page """
url_to_parse = self._url
while True:
self._soup: Tag = self._parser.parse(url=url_to_parse)
listings: List[Tag] = self._soup.find_all(
"div", attrs={"class": "a-row a-spacing-mini olpOffer"})
for row in listings:
price: float = self._parse_price(row)
seller: str = self._parse_seller(row)
rating: int = self._parse_rating(row)
no_of_ratings: int = self._parse_no_of_ratings(row)
condition: Condition = Condition(self._parse_condition(row))
self._product_listings.append(
ProductListing(seller=seller,
price=price,
rating=rating,
total_ratings=no_of_ratings,
condition=condition))
url_to_parse = self._parse_next_link(self._soup)
if url_to_parse is None:
break
# Wait for 1 sec before scrapping next page
time.sleep(1)
return self._product_listings
def __init__(self, target: str, directory: str):
self.target = target
self.directory = directory
self.aggression = False
self.timeout = 30
self.allow_redirect = True
self.max_threads = 8
self._headers = {"user-agent": fake_user_agent()}
self._cond_parser = Condition()
# threading lock variable
self._results = []
self._implies = set()
self._excludes = set()
def process_input(self):
""" Read the product information from the file """
output_file = self._get_output_file(self._input_file)
with open(output_file, mode="w") as output_file:
with open(self._input_file, mode="r") as infile:
for line in infile:
line = line.strip()
line_values = line.split()
asin: str = line_values[0]
condition: int = int(line_values[1])
print(f"Processing ASIN: {asin}")
product_url = Amazon.PRODUCT_URL.format(asin)
product_listing_url = Amazon.LISTING_URL.format(asin)
print(f"Parsing product")
product: Product = ProductParser(product_url).parse()
product_listing_parser: ProductListingParser = \
ProductListingParser(product_listing_url)
print(f"Parsing product listings")
product_listings: List[
ProductListing] = product_listing_parser.parse()
print(f"Repricing")
repricer: Repricer = Repricer(product, product_listings)
my_product_listing: ProductListing = product_listing_parser.my_listing
repricer.rating_filter = self._target_rating
repricer.condition_filter = Condition(condition)
price = repricer.reprice(my_product_listing)
profit = repricer.calculate_profit(
price, my_product_listing.shipping)
print(f"Mew Price: {price:.2f}")
print(f"Profit: {profit:.2f}")
# Output to file
if profit > self._min_profit:
output_file.write(f"{product}\n")
output_file.write(f"{price:.2f}\n\n")
else:
self._unprofitable.append(str(product))
print(f"Completed!!!\n\n")