def download_picture(self, picture_url: str, album_folder: str) -> None:
"""
Download picture.
:param picture_url: picture url
:param album_folder: folder path
"""
try:
if picture_url.startswith('//'):
picture_url = picture_url.replace('//', '', 1)
picture_name = picture_url.rsplit('/', 1)[1]
picture_path = os.path.join(album_folder, picture_name)
if not os.path.exists(picture_path):
logger.info(f'Start downloading: {picture_url}')
retry = 1
response = requests.get(picture_url,
stream=True,
timeout=self.timeout)
while response.status_code != 200 and retry <= self.retries:
logger.warning(f'{retry}º Retry: {picture_name}')
response = requests.get(picture_url,
stream=True,
timeout=self.timeout)
retry += 1
if len(response.content) > 0:
with open(picture_path, 'wb') as image:
image.write(response.content)
logger.log(5, f'Completed download of: {picture_name}')
else:
raise Exception('Zero content')
else:
logger.warning(f'Picture already exists: {picture_name} ')
except Exception as e:
logger.error(f'Failed to download picture: {picture_url}\n{e}')
def download_picture(self, picture_url: str, album_folder: Path) -> None:
"""
Download picture.
:param picture_url: picture url
:param album_folder: album folder path
"""
try:
picture_url = normalize_url(picture_url)
picture_name = picture_url.rsplit('/', 1)[1]
picture_path = Path.joinpath(album_folder, picture_name)
if not Path.exists(picture_path):
logger.info(f'Start downloading: {picture_url}')
retry = 1
response = requests.get(picture_url,
stream=True,
timeout=self.timeout)
while response.status_code != 200 and retry <= self.retries:
logger.warning(f'{retry}º Retry: {picture_name}')
response = requests.get(picture_url,
stream=True,
timeout=self.timeout)
retry += 1
if retry > self.retries:
raise Exception('Reached maximum number of retries')
if len(response.content) > 0:
with picture_path.open('wb') as image:
image.write(response.content)
logger.log(5, f'Completed download of: {picture_name}')
else:
raise Exception('Zero content')
else:
logger.warning(f'Picture already exists: {picture_name} ')
except Exception as e:
logger.error(f'Failed to download picture: {picture_url}\n{e}')
def fetch_info(self) -> bool:
"""
Fetch album information.
:return: bool - true if there are no error otherwise false
"""
logger.log(5, 'Fetching album information...')
response = requests.post(
'https://members.luscious.net/graphql/nobatch/?operationName=AlbumGet',
json=album_info_query(str(self.id_))).json()
data = response['data']['album']['get']
if 'errors' in data:
logger.error(
f'Something wrong with album: {self.id_}\nErrors: {data["errors"]}'
)
logger.warning('Skipping...')
return False
self.title = data['title']
self.author = data['created_by']['display_name']
self.number_of_pictures = data['number_of_pictures']
self.number_of_animated_pictures = data['number_of_animated_pictures']
self.info.update({
'slug':
data.get('slug', self.title),
'language':
data.get('language', {}).get('title', ''),
'tags': [tag.get('text', '') for tag in data.get('tags', {})],
'genres':
[genre.get('title', '') for genre in data.get('genres', {})],
'audiences': [
audience.get('title', '')
for audience in data.get('audiences', {})
],
})
return True
def start(album_url):
start_time = time.time()
album_id = get_album_id(album_url)
if not album_id:
logger.warning(f'No album id. Skipping...')
return
album_info = get_album_info(album_id)
show_album_info(album_info)
directory = get_config_setting('directory')
pool_size = get_config_setting('pool')
picture_page_urls = get_pictures_urls(album_id)
album_name = re.sub('[^\w\-_\. ]', '_', album_info['title'])
create_folder(f'{directory}{album_name}/')
logger.info('Starting download pictures.')
pool = mp.Pool(pool_size)
pool.starmap(download_picture,
zip(picture_page_urls, repeat(directory), repeat(album_name)))
end_time = time.time()
logger.info(
f'Album > {album_name} < Download completed {len(picture_page_urls)} pictures has saved.'
)
logger.info(f'Finished download in {end_time-start_time:.2f}')
list_organizer(album_url)
def show_album_info(album):
try:
logger.log(
5,
f'Album Name: {album["title"]} - with {album["number_of_pictures"]} pictures.'
)
except Exception as e:
logger.warning(f'Failed to print album information.\n{e}')
def get_config_data() -> dict:
"""
Load and return config.json data.
:return: dictionary
"""
try:
with get_root_path().joinpath('config.json').open() as config:
data = json.load(config)
return data
except Exception as e:
logger.warning(f'Something went wrong loading config file: {e}')
return {}
def normalize_args(args: Namespace) -> Namespace:
"""
Fix possible args inconsistencies.
:param args: Namespace
:return: Fixed Namespace
"""
if args.threads <= 0:
args.threads = os.cpu_count()
if not args.threads:
logger.warning(
'It was not possible to determine the number of CPUs in your system. '
'Only one will be used, this will decrease the amount of downloads.'
)
args.threads = 1
if args.page <= 0:
args.page = 1
if args.max_pages <= 0:
args.max_pages = 1
if args.page > args.max_pages:
args.max_pages = args.page
if args.only_favorites and not args.user_inputs:
logger.warn(
f"You're passing --favorites/-f flag without any user input.")
args.only_favorites = False
if args.gen_pdf and not args.album_inputs and not args.user_inputs and not args.search_download:
logger.warn(
f"You're passing --pdf flag without any album/user input or search download."
)
args.gen_pdf = False
args.keyword = args.keyword.strip() if args.keyword else None
if args.album_inputs:
inputs = inputs_string_to_list(args.album_inputs)
args.albums_ids = extract_ids_from_list(inputs, extract_album_id)
else:
args.albums_ids = None
if args.user_inputs:
inputs = inputs_string_to_list(args.user_inputs)
args.users_ids = extract_ids_from_list(inputs, extract_user_id)
else:
args.users_ids = None
return args
def fetch_info(self) -> bool:
"""
Fetch user information.
:return: bool - true if there are no error otherwise false
"""
logger.log(5, 'Fetching user information...')
response = requests.post(
'https://members.luscious.net/graphql/nobatch/?operationName=ProfileGet',
json=user_info_query(str(self.id_))).json()
data = response['data']['userprofile']['get']
if "errors" in data:
logger.error(
f'Something wrong with user: {self.id_}\nErrors: {data["errors"]}'
)
logger.warning('Skipping...')
return False
self.name = data['user']['name']
self.number_of_albums = data['number_of_albums']
self.number_of_favorites = data['number_of_favorite_albums']
return True
def fetch_info(self) -> bool:
"""
Fetch album information.
:return: bool - true if there are no error otherwise false
"""
logger.log(5, 'Fetching album information...')
response = requests.post(
'https://members.luscious.net/graphql/nobatch/?operationName=AlbumGet',
json=album_info_query(str(self.id_))).json()
data = response['data']['album']['get']
if 'errors' in data:
logger.error(
f'Something wrong with album: {self.id_}\nErrors: {data["errors"]}'
)
logger.warning('Skipping...')
return False
self.title = data['title']
self.author = data['created_by']['display_name']
self.number_of_pictures = data['number_of_pictures']
self.number_of_animated_pictures = data['number_of_animated_pictures']
return True
def download_picture(picture_url, directory, album_name):
try:
picture_name = picture_url.rsplit('/', 1)[1]
picture_path = f'{directory}{album_name}/{picture_name}'
if not (os.path.exists(picture_path)):
logger.info(f'Start downloading: {picture_url}')
retries = 1
res = requests.get(picture_url, stream=True)
while res.status_code != 200 and retries <= 5:
logger.warning(f'{retries}º Retry: {picture_name}')
res = requests.get(picture_url, stream=True)
if len(res.content) > 0:
with open(picture_path, 'wb') as image:
image.write(res.content)
logger.log(5, f'Completed download of: {picture_name}')
else:
raise Exception('Zero content')
else:
logger.warning(f'Picture: {picture_name} already exist.')
except Exception as e:
logger.error(f'Failed to download picture: {picture_url}\n{e}')
