def get_pictures_urls(album_id): logger.log(5, 'Fetching album pictures...') n = 1 raw_data = [] while True: response = requests.post( 'https://members.luscious.net/graphql/nobatch/?operationName=AlbumListOwnPictures', json={ "id": 7, "operationName": "AlbumListOwnPictures", "query": "query AlbumListOwnPictures($input: PictureListInput!) {picture {list(input: $input) {info " "{...FacetCollectionInfo} items {...PictureStandardWithoutAlbum}}}} fragment FacetCollectionInfo on " "FacetCollectionInfo {page has_next_page has_previous_page total_items total_pages items_per_page " "url_complete} fragment PictureStandardWithoutAlbum on Picture {url_to_original url_to_video url}", "variables": { "input": { "filters": [{ "name": "album_id", "value": album_id }], "display": "rating_all_time", "page": n } } }).json() raw_data.append(response['data']['picture']['list']['items']) n += 1 if not response['data']['picture']['list']['info']['has_next_page']: break data = [obj['url_to_original'] for arr in raw_data for obj in arr] logger.info(f'Total of {len(data)} links found.') return data
def download_picture(self, picture_url: str, album_folder: str) -> None: """ Download picture. :param picture_url: picture url :param album_folder: folder path """ try: if picture_url.startswith('//'): picture_url = picture_url.replace('//', '', 1) picture_name = picture_url.rsplit('/', 1)[1] picture_path = os.path.join(album_folder, picture_name) if not os.path.exists(picture_path): logger.info(f'Start downloading: {picture_url}') retry = 1 response = requests.get(picture_url, stream=True, timeout=self.timeout) while response.status_code != 200 and retry <= self.retries: logger.warning(f'{retry}º Retry: {picture_name}') response = requests.get(picture_url, stream=True, timeout=self.timeout) retry += 1 if len(response.content) > 0: with open(picture_path, 'wb') as image: image.write(response.content) logger.log(5, f'Completed download of: {picture_name}') else: raise Exception('Zero content') else: logger.warning(f'Picture already exists: {picture_name} ') except Exception as e: logger.error(f'Failed to download picture: {picture_url}\n{e}')
def download_picture(self, picture_url: str, album_folder: Path) -> None: """ Download picture. :param picture_url: picture url :param album_folder: album folder path """ try: picture_url = normalize_url(picture_url) picture_name = picture_url.rsplit('/', 1)[1] picture_path = Path.joinpath(album_folder, picture_name) if not Path.exists(picture_path): logger.info(f'Start downloading: {picture_url}') retry = 1 response = requests.get(picture_url, stream=True, timeout=self.timeout) while response.status_code != 200 and retry <= self.retries: logger.warning(f'{retry}º Retry: {picture_name}') response = requests.get(picture_url, stream=True, timeout=self.timeout) retry += 1 if retry > self.retries: raise Exception('Reached maximum number of retries') if len(response.content) > 0: with picture_path.open('wb') as image: image.write(response.content) logger.log(5, f'Completed download of: {picture_name}') else: raise Exception('Zero content') else: logger.warning(f'Picture already exists: {picture_name} ') except Exception as e: logger.error(f'Failed to download picture: {picture_url}\n{e}')
def create_folder(directory): try: if not os.path.exists(directory): os.makedirs(directory) logger.info(f'Album folder created: {directory}') else: logger.warn(f'Album folder {directory} already exist.') except OSError: logger.error(f'Creating directory: {directory}\n{e}')
def download(self, downloader: Downloader) -> None: """ Start album download. :param downloader: Downloder object """ logger.info(f'Starting album download: {self.title}') if downloader: downloader.download(self.title, self.pictures) else: logger.critical( f'Downloader not set in album: {self.id_} | {self.title}')
def download(self, downloader: Downloader, album_folder: Path) -> None: """ Start album download. :param downloader: Downloader object :param album_folder: album folder """ logger.info(f'Starting album download: {self.title}') if downloader: downloader.download(self.pictures, album_folder) logger.info(f'Album download completed: {self.title}') else: logger.critical( f'Downloader not set in album: {self.id_} | {self.title}')
def create_folder(directory: Path) -> None: """ Creates folder in the specified path. :param directory: folder path """ try: if not Path.exists(directory): Path.mkdir(directory, parents=True, exist_ok=True) logger.info(f'Album folder created in: {directory}') else: logger.warn(f'Album folder already exist in: {directory}') except Exception as e: logger.error(f'Create folder: {e}')
def create_folder(directory: str) -> None: """ Creates folder in the specified path. :param directory: folder path """ try: if not os.path.exists(directory): os.makedirs(directory, exist_ok=True) logger.info(f'Album folder created in: {directory}') else: logger.warn(f'Album folder already exist in: {directory}') except OSError: logger.error(f'Creating directory in: {directory}')
def generate_pdf(output_dir: Path, formmatted_name: str, album_folder: Path, rm_origin_dir=False) -> None: """ Create pdf file containing album pictures [jpg,jpeg]. :param output_dir: output folder path :param formmatted_name: formmatted album name :param album_folder: album folder path :param rm_origin_dir: indicates whether the source folder will be deleted """ try: from PIL import Image logger.info('Generating album pdf file...') pictures_path_list = [] for file_name in album_folder.iterdir(): if file_name.suffix.lower() not in ['.jpg', '.jpeg', '.png']: continue picture_path = Path.joinpath(album_folder, file_name) if picture_path.is_dir(): continue pictures_path_list.append(picture_path) pictures = [] if len(pictures_path_list) > 0: for picture_path in pictures_path_list: img = Image.open(picture_path) if picture_path.suffix.lower() == '.png' or img.mode == 'RGBA': img = img.convert('RGB') pictures.append(img) if len(pictures) == 0: raise Exception('Pictures list is empty, probably has no valid images [jpg, jpeg, png]') pdf_filename = f'{formmatted_name}.pdf' pdf_path = Path.joinpath(output_dir, pdf_filename) logger.info(f'Adding {len(pictures)} pictures to pdf...') pictures[0].save(pdf_path, save_all=True, append_images=pictures[1:]) logger.log(5, f'Album PDF saved to: {output_dir}') for img in pictures: img.close() if rm_origin_dir: shutil.rmtree(album_folder, ignore_errors=True) logger.log(5, f'Album {formmatted_name} folder deleted.') except ImportError: logger.error('Please install Pillow package by using pip.') except Exception as e: logger.error(f'Failed to generate album pdf: {e}')
def fetch_pictures(self) -> None: """Fetch album pictures.""" logger.log(5, 'Fetching album pictures...') page = 1 while True: response = requests.post( 'https://members.luscious.net/graphql/nobatch/?operationName=AlbumListOwnPictures', json=album_list_pictures_query(str(self.id_), page)).json() self.pictures.extend([ picture['url_to_original'] for picture in response['data']['picture']['list']['items'] ]) page += 1 if not response['data']['picture']['list']['info']['has_next_page']: break logger.info(f'Total of {len(self.pictures)} links found.')
def download(self, urls: list[str], album_folder: Path) -> None: """ Start download process. :param urls: list of image URLs :param album_folder: album folder """ start_time = time.time() create_folder(album_folder) pool = mp.Pool(self.threads) pool.starmap(self.download_picture, zip(urls, repeat(album_folder))) end_time = time.time() logger.info( f'Finished in {time.strftime("%H:%M:%S", time.gmtime(end_time - start_time))}' ) if self.delay: time.sleep(self.delay)
def download_picture(picture_url, directory, album_name): try: picture_name = picture_url.rsplit('/', 1)[1] picture_path = f'{directory}{album_name}/{picture_name}' if not (os.path.exists(picture_path)): logger.info(f'Start downloading: {picture_url}') retries = 1 res = requests.get(picture_url, stream=True) while res.status_code != 200 and retries <= 5: logger.warning(f'{retries}º Retry: {picture_name}') res = requests.get(picture_url, stream=True) if len(res.content) > 0: with open(picture_path, 'wb') as image: image.write(res.content) logger.log(5, f'Completed download of: {picture_name}') else: raise Exception('Zero content') else: logger.warning(f'Picture: {picture_name} already exist.') except Exception as e: logger.error(f'Failed to download picture: {picture_url}\n{e}')
def start(album_url): start_time = time.time() album_id = get_album_id(album_url) if not album_id: logger.warning(f'No album id. Skipping...') return album_info = get_album_info(album_id) show_album_info(album_info) directory = get_config_setting('directory') pool_size = get_config_setting('pool') picture_page_urls = get_pictures_urls(album_id) album_name = re.sub('[^\w\-_\. ]', '_', album_info['title']) create_folder(f'{directory}{album_name}/') logger.info('Starting download pictures.') pool = mp.Pool(pool_size) pool.starmap(download_picture, zip(picture_page_urls, repeat(directory), repeat(album_name))) end_time = time.time() logger.info( f'Album > {album_name} < Download completed {len(picture_page_urls)} pictures has saved.' ) logger.info(f'Finished download in {end_time-start_time:.2f}') list_organizer(album_url)
def fetch_albums(self, only_favorites=False): """Fetch user albums.""" logger.log( 5, f'Fetching user {"favorites" if only_favorites else "albums"}...') n = 1 while True: logger.info( f'Fetching user {"favorites" if only_favorites else "albums"} page: {n}...' ) json_query = user_favorites_query if only_favorites else user_albums_query response = requests.post( 'https://members.luscious.net/graphql/nobatch/?operationName=AlbumList', json=json_query(str(self.id_), n)).json() self.albums_ids.extend([ album['id'] for album in response['data']['album']['list']['items'] ]) n += 1 if not response['data']['album']['list']['info']['has_next_page']: break logger.info(f'Total of {len(self.albums_ids)} ids found.')
def download(self, album_title: str, urls: List[str]) -> None: """ Start download process. :param album_title: Album title :param urls: list of image URLs """ start_time = time.time() album_name = re.sub(r'[^\w\-_\. ]', '_', album_title) album_folder = os.path.join(self.output_dir, album_name) create_folder(album_folder) pool = mp.Pool(self.threads) pool.starmap(self.download_picture, zip(urls, repeat(album_folder))) end_time = time.time() logger.info(f'Album download completed: {album_title}') logger.info( f'Finished in {time.strftime("%H:%M:%S", time.gmtime(end_time - start_time))}' ) if self.delay: time.sleep(self.delay)
def info() -> None: """Show package version.""" logger.info(f'Luscious Downloader version: {__version__}')