def get_pictures_urls(album_id):
logger.log(5, 'Fetching album pictures...')
n = 1
raw_data = []
while True:
response = requests.post(
'https://members.luscious.net/graphql/nobatch/?operationName=AlbumListOwnPictures',
json={
"id":
7,
"operationName":
"AlbumListOwnPictures",
"query":
"query AlbumListOwnPictures($input: PictureListInput!) {picture {list(input: $input) {info "
"{...FacetCollectionInfo} items {...PictureStandardWithoutAlbum}}}} fragment FacetCollectionInfo on "
"FacetCollectionInfo {page has_next_page has_previous_page total_items total_pages items_per_page "
"url_complete} fragment PictureStandardWithoutAlbum on Picture {url_to_original url_to_video url}",
"variables": {
"input": {
"filters": [{
"name": "album_id",
"value": album_id
}],
"display": "rating_all_time",
"page": n
}
}
}).json()
raw_data.append(response['data']['picture']['list']['items'])
n += 1
if not response['data']['picture']['list']['info']['has_next_page']:
break
data = [obj['url_to_original'] for arr in raw_data for obj in arr]
logger.info(f'Total of {len(data)} links found.')
return data
def download_picture(self, picture_url: str, album_folder: str) -> None:
"""
Download picture.
:param picture_url: picture url
:param album_folder: folder path
"""
try:
if picture_url.startswith('//'):
picture_url = picture_url.replace('//', '', 1)
picture_name = picture_url.rsplit('/', 1)[1]
picture_path = os.path.join(album_folder, picture_name)
if not os.path.exists(picture_path):
logger.info(f'Start downloading: {picture_url}')
retry = 1
response = requests.get(picture_url,
stream=True,
timeout=self.timeout)
while response.status_code != 200 and retry <= self.retries:
logger.warning(f'{retry}º Retry: {picture_name}')
response = requests.get(picture_url,
stream=True,
timeout=self.timeout)
retry += 1
if len(response.content) > 0:
with open(picture_path, 'wb') as image:
image.write(response.content)
logger.log(5, f'Completed download of: {picture_name}')
else:
raise Exception('Zero content')
else:
logger.warning(f'Picture already exists: {picture_name} ')
except Exception as e:
logger.error(f'Failed to download picture: {picture_url}\n{e}')
def download_picture(self, picture_url: str, album_folder: Path) -> None:
"""
Download picture.
:param picture_url: picture url
:param album_folder: album folder path
"""
try:
picture_url = normalize_url(picture_url)
picture_name = picture_url.rsplit('/', 1)[1]
picture_path = Path.joinpath(album_folder, picture_name)
if not Path.exists(picture_path):
logger.info(f'Start downloading: {picture_url}')
retry = 1
response = requests.get(picture_url,
stream=True,
timeout=self.timeout)
while response.status_code != 200 and retry <= self.retries:
logger.warning(f'{retry}º Retry: {picture_name}')
response = requests.get(picture_url,
stream=True,
timeout=self.timeout)
retry += 1
if retry > self.retries:
raise Exception('Reached maximum number of retries')
if len(response.content) > 0:
with picture_path.open('wb') as image:
image.write(response.content)
logger.log(5, f'Completed download of: {picture_name}')
else:
raise Exception('Zero content')
else:
logger.warning(f'Picture already exists: {picture_name} ')
except Exception as e:
logger.error(f'Failed to download picture: {picture_url}\n{e}')
def create_folder(directory):
try:
if not os.path.exists(directory):
os.makedirs(directory)
logger.info(f'Album folder created: {directory}')
else:
logger.warn(f'Album folder {directory} already exist.')
except OSError:
logger.error(f'Creating directory: {directory}\n{e}')
def download(self, downloader: Downloader) -> None:
"""
Start album download.
:param downloader: Downloder object
"""
logger.info(f'Starting album download: {self.title}')
if downloader:
downloader.download(self.title, self.pictures)
else:
logger.critical(
f'Downloader not set in album: {self.id_} | {self.title}')
def download(self, downloader: Downloader, album_folder: Path) -> None:
"""
Start album download.
:param downloader: Downloader object
:param album_folder: album folder
"""
logger.info(f'Starting album download: {self.title}')
if downloader:
downloader.download(self.pictures, album_folder)
logger.info(f'Album download completed: {self.title}')
else:
logger.critical(
f'Downloader not set in album: {self.id_} | {self.title}')
def create_folder(directory: Path) -> None:
"""
Creates folder in the specified path.
:param directory: folder path
"""
try:
if not Path.exists(directory):
Path.mkdir(directory, parents=True, exist_ok=True)
logger.info(f'Album folder created in: {directory}')
else:
logger.warn(f'Album folder already exist in: {directory}')
except Exception as e:
logger.error(f'Create folder: {e}')
def create_folder(directory: str) -> None:
"""
Creates folder in the specified path.
:param directory: folder path
"""
try:
if not os.path.exists(directory):
os.makedirs(directory, exist_ok=True)
logger.info(f'Album folder created in: {directory}')
else:
logger.warn(f'Album folder already exist in: {directory}')
except OSError:
logger.error(f'Creating directory in: {directory}')
def generate_pdf(output_dir: Path, formmatted_name: str, album_folder: Path, rm_origin_dir=False) -> None:
"""
Create pdf file containing album pictures [jpg,jpeg].
:param output_dir: output folder path
:param formmatted_name: formmatted album name
:param album_folder: album folder path
:param rm_origin_dir: indicates whether the source folder will be deleted
"""
try:
from PIL import Image
logger.info('Generating album pdf file...')
pictures_path_list = []
for file_name in album_folder.iterdir():
if file_name.suffix.lower() not in ['.jpg', '.jpeg', '.png']:
continue
picture_path = Path.joinpath(album_folder, file_name)
if picture_path.is_dir():
continue
pictures_path_list.append(picture_path)
pictures = []
if len(pictures_path_list) > 0:
for picture_path in pictures_path_list:
img = Image.open(picture_path)
if picture_path.suffix.lower() == '.png' or img.mode == 'RGBA':
img = img.convert('RGB')
pictures.append(img)
if len(pictures) == 0:
raise Exception('Pictures list is empty, probably has no valid images [jpg, jpeg, png]')
pdf_filename = f'{formmatted_name}.pdf'
pdf_path = Path.joinpath(output_dir, pdf_filename)
logger.info(f'Adding {len(pictures)} pictures to pdf...')
pictures[0].save(pdf_path, save_all=True, append_images=pictures[1:])
logger.log(5, f'Album PDF saved to: {output_dir}')
for img in pictures:
img.close()
if rm_origin_dir:
shutil.rmtree(album_folder, ignore_errors=True)
logger.log(5, f'Album {formmatted_name} folder deleted.')
except ImportError:
logger.error('Please install Pillow package by using pip.')
except Exception as e:
logger.error(f'Failed to generate album pdf: {e}')
def fetch_pictures(self) -> None:
"""Fetch album pictures."""
logger.log(5, 'Fetching album pictures...')
page = 1
while True:
response = requests.post(
'https://members.luscious.net/graphql/nobatch/?operationName=AlbumListOwnPictures',
json=album_list_pictures_query(str(self.id_), page)).json()
self.pictures.extend([
picture['url_to_original']
for picture in response['data']['picture']['list']['items']
])
page += 1
if not response['data']['picture']['list']['info']['has_next_page']:
break
logger.info(f'Total of {len(self.pictures)} links found.')
def download(self, urls: list[str], album_folder: Path) -> None:
"""
Start download process.
:param urls: list of image URLs
:param album_folder: album folder
"""
start_time = time.time()
create_folder(album_folder)
pool = mp.Pool(self.threads)
pool.starmap(self.download_picture, zip(urls, repeat(album_folder)))
end_time = time.time()
logger.info(
f'Finished in {time.strftime("%H:%M:%S", time.gmtime(end_time - start_time))}'
)
if self.delay:
time.sleep(self.delay)
def download_picture(picture_url, directory, album_name):
try:
picture_name = picture_url.rsplit('/', 1)[1]
picture_path = f'{directory}{album_name}/{picture_name}'
if not (os.path.exists(picture_path)):
logger.info(f'Start downloading: {picture_url}')
retries = 1
res = requests.get(picture_url, stream=True)
while res.status_code != 200 and retries <= 5:
logger.warning(f'{retries}º Retry: {picture_name}')
res = requests.get(picture_url, stream=True)
if len(res.content) > 0:
with open(picture_path, 'wb') as image:
image.write(res.content)
logger.log(5, f'Completed download of: {picture_name}')
else:
raise Exception('Zero content')
else:
logger.warning(f'Picture: {picture_name} already exist.')
except Exception as e:
logger.error(f'Failed to download picture: {picture_url}\n{e}')
def start(album_url):
start_time = time.time()
album_id = get_album_id(album_url)
if not album_id:
logger.warning(f'No album id. Skipping...')
return
album_info = get_album_info(album_id)
show_album_info(album_info)
directory = get_config_setting('directory')
pool_size = get_config_setting('pool')
picture_page_urls = get_pictures_urls(album_id)
album_name = re.sub('[^\w\-_\. ]', '_', album_info['title'])
create_folder(f'{directory}{album_name}/')
logger.info('Starting download pictures.')
pool = mp.Pool(pool_size)
pool.starmap(download_picture,
zip(picture_page_urls, repeat(directory), repeat(album_name)))
end_time = time.time()
logger.info(
f'Album > {album_name} < Download completed {len(picture_page_urls)} pictures has saved.'
)
logger.info(f'Finished download in {end_time-start_time:.2f}')
list_organizer(album_url)
def fetch_albums(self, only_favorites=False):
"""Fetch user albums."""
logger.log(
5,
f'Fetching user {"favorites" if only_favorites else "albums"}...')
n = 1
while True:
logger.info(
f'Fetching user {"favorites" if only_favorites else "albums"} page: {n}...'
)
json_query = user_favorites_query if only_favorites else user_albums_query
response = requests.post(
'https://members.luscious.net/graphql/nobatch/?operationName=AlbumList',
json=json_query(str(self.id_), n)).json()
self.albums_ids.extend([
album['id']
for album in response['data']['album']['list']['items']
])
n += 1
if not response['data']['album']['list']['info']['has_next_page']:
break
logger.info(f'Total of {len(self.albums_ids)} ids found.')
def download(self, album_title: str, urls: List[str]) -> None:
"""
Start download process.
:param album_title: Album title
:param urls: list of image URLs
"""
start_time = time.time()
album_name = re.sub(r'[^\w\-_\. ]', '_', album_title)
album_folder = os.path.join(self.output_dir, album_name)
create_folder(album_folder)
pool = mp.Pool(self.threads)
pool.starmap(self.download_picture, zip(urls, repeat(album_folder)))
end_time = time.time()
logger.info(f'Album download completed: {album_title}')
logger.info(
f'Finished in {time.strftime("%H:%M:%S", time.gmtime(end_time - start_time))}'
)
if self.delay:
time.sleep(self.delay)
def info() -> None:
"""Show package version."""
logger.info(f'Luscious Downloader version: {__version__}')
