def miner(self):
# Miner's track : We land on artwork listings page. We pick the listings from there.
# We pick the Sellers and Artists from artwork pages.
# From artwork pages we fetch the artwork for sale for artists listed.
self.get_artist_listings()
# print(kazoart.artist_listings)
# print(self.artist_listings)
# print("ARTIST LISTINGS")
# print(len(self.artist_listings))
# time.sleep(10)
# That the pages where we discarded the links can be visited as well
for link in self.first_prod_list:
visited.discard(link)
self.get_artwork_listings_master()
# get_artwork_listings_master -> get_artwork_listings_slave -> get_artist_data -> write_artist_data
# So we're done with artist data.
# print(f"ARTWORK LISTINGS, {len(self.artwork_listings)}")
# print(len(self.artwork_listings))
# time.sleep(10)
self.get_artwork_data_master()
# DATA COLLECTION COMPLETED FOR THIS MODULE.
# DOWNLOADING IMAGES NOW.
TheMiner.sir_image_manager()
def miner(self):
self.get_artist_listings()
print(len(self.artist_listings))
self.get_artwork_listings_master()
# get_artwork_listings_master -> get_artwork_listings_slave -> get_artist_data -> write_artist_data
# So we're done with artist data.
print(len(self.artwork_listings))
self.get_artwork_data_master()
# DATA COLLECTION COMPLETED FOR THIS MODULE.
# DOWNLOADING IMAGES NOW.
TheMiner.sir_image_manager()
def miner(self):
# self.get_artist_listings()
# print(kazoart.artist_listings)
# print(len(self.artist_listings))
self.get_artwork_listings_master()
# get_artwork_listings_master -> get_artwork_listings_slave
# We're still not done with artist data
print(len(self.artwork_listings))
self.get_artwork_data_master()
# DATA COLLECTION COMPLETED FOR THIS MODULE.
# DOWNLOADING IMAGES NOW.
print("downloading images now.")
TheMiner.sir_image_manager(chunk_size=100)
def main():
start = time.perf_counter()
# Creating SELLER_INFO
sellers = db.Sellers()
sellers.read_data_sellers()
# Creating ARTIST_INFO
artists = db.Artist()
artists.read_artist_data()
artsperpainters = Website(
'https://www.artsper.com',
'https://www.artsper.com/us/contemporary-artists/youngtalents/painters?',
"ARTSPER")
a_m = Artsper(artsperpainters)
a_m.artsper_mine()
finish = time.perf_counter()
print(
f"Lap Completed in {round(finish - start, 2)}, seconds.\n Starting sculptures"
)
artspersculptors = Website(
'https://www.artsper.com',
'https://www.artsper.com/us/contemporary-artists/youngtalents/sculptors-artists',
"ARTSPER")
a_m = Artsper(artspersculptors)
a_m.artsper_mine()
finish = time.perf_counter()
print(
f"Lap Completed in {round(finish - start, 2)}, seconds.\n Downloading and updating images"
)
TheMiner.sir_image_manager()
finish = time.perf_counter()
print(f"Finished in {round(finish - start, 2)}, seconds")