def save_collections_artworks(self, collections, dir_path):
result = []
for collection in collections:
files = self.save_collection_artworks(collection, dir_path)
if not files:
continue
result.append(files)
return utils.counter(result)
def save_artists(self, artist_ids, dir_path):
print(f"\nthere are {len(artist_ids)} artists\n")
result = []
for id in artist_ids:
files = self.save_artist(id, dir_path)
if not files:
continue
result.append(files)
return utils.counter(result)
def save_users_artworks(self, user_ids, dir_path):
print(f'\nthere are {len(user_ids)} users\n')
result = []
for user in user_ids:
files = self.save_user_artworks(user, dir_path)
if not files:
continue
result.append(files)
return utils.counter(result)
def save_ranking_artworks(self, dir_path, order='popular-1-month', type='visual-art', content='all', category='all', limit=30):
print(f'download {order} {content} {category} ranking\n')
dir_path = utils.make_dir(dir_path, f'{order} {content} {category} ranking')
artworks = self.ranking_artworks(order, type, content, category, limit, dir_path)
if not artworks:
print(f'{order} {content} {category} ranking is up-to-date\n')
return
with ThreadPool(self.threads) as pool:
files = pool.map(partial(self.save_artwork, dir_path), artworks)
print(f'\ndownload for {order} {content} {category} ranking completed\n')
combined_files = utils.counter(files)
utils.set_files_mtime(combined_files['name'], dir_path)
return combined_files
def save_user_artworks(self, user_id, dir_path):
print(f'download artworks for user {user_id}\n')
dir_path = utils.make_dir(dir_path, user_id)
artworks = self.user_artworks(user_id, dir_path)
if not artworks:
print(f'user {user_id} is up-to-date\n')
return
with ThreadPool(self.threads) as pool:
files = pool.map(partial(self.save_artwork, dir_path), artworks)
print(f'\ndownload for user {user_id} completed\n')
combined_files = utils.counter(files)
utils.set_files_mtime(combined_files['name'], dir_path)
return combined_files
def save_artist(self, artist_id, dir_path):
artist_name = self.artist(artist_id)["name"]
print(f"download for artist {artist_name} begins\n")
dir_path = utils.make_dir(dir_path, artist_id)
artworks = self.artist_artworks(artist_id, dir_path)
if not artworks:
print(f"artist {artist_name} is up-to-date\n")
return
with ThreadPool(self.threads) as pool:
files = pool.map(partial(self.save_artwork, dir_path), artworks)
print(f"\ndownload for artist {artist_name} completed\n")
combined_files = utils.counter(files)
utils.file_mtimes(combined_files["names"], dir_path)
return combined_files
def save_artist(self, artist_id, dir_path, stop=None):
gallery = self.gallery(artist_id)
artist_name = gallery["artist_name"]
print(f"download for author {artist_name} begins\n")
dir_path = utils.make_dir(dir_path, artist_name)
artwork_urls = self.artist_artworks(artist_id, stop)
if not artwork_urls:
print(f"author {artist_name} is up-to-date\n")
return
with ThreadPool(self.threads) as pool:
files = pool.map(partial(self.save_artwork, dir_path), artwork_urls)
print(f"\ndownload for author {artist_name} completed\n")
combined_files = utils.counter(files)
utils.set_files_mtime(combined_files["name"], dir_path)
return combined_files
def download_artists(api, config):
start_time = time.time()
print(f"\nthere are {len(config.artists)} artists\n")
result = []
for artist_id, artwork_url in config.artists.items():
files = api.save_artist(artist_id, config.save_dir, artwork_url)
if not files:
continue
config.update_artist(artist_id, files["url"][0])
result.append(files)
result = utils.counter(result)
duration = time.time() - start_time
size_mb = result["size"] / 1048576
print("\nSUMMARY")
print("---------------------------------")
print(f"time elapsed:\t{duration:.4f} seconds")
print(f"total size:\t{size_mb:.4f} MB")
print(f"total artworks:\t{result['count']} artworks")
print(f"download speed:\t{(size_mb / duration):.4f} MB/s")
def save_collection_artworks(self, collection, dir_path):
collection_metadata = self.collection_metadata(collection)
collection_name = collection_metadata["name"]
collection_owner = collection_metadata["owner"]["username"]
amount = collection_metadata["size"]
print(
f'download {amount} artworks for collection {collection_name} by {collection_owner}\n'
)
dir_path = utils.make_dir(dir_path, collection_name)
artworks = self.collection_artworks(collection, dir_path)
if not artworks:
print(f'collection {collection_name} is up-to-date\n')
return
with ThreadPool(self.threads) as pool:
files = pool.map(partial(self.save_artwork, dir_path), artworks)
print(f'\ndownload for collection {collection_name} completed\n')
combined_files = utils.counter(files)
utils.set_files_mtime(combined_files['name'], dir_path)
return combined_files