def chapter_download_get_link_only(self):
"""
:return:
Validate page_list: if unable to get or is null => Return default chapter_result
Append all pages to threading and run
Validate chapter after downloading
Import result to mongodb
"""
if self.page_list == list():
return self.chapter_result
page_number = int(self.start_page)
print "\nDOWNLOADING Story : {0} - chapter-{1} - site_url {2} - {3} pages"\
.format(self.story.story_name, self.chapter_name, self.story.site_url, len(self.page_list))
# Enable this for debug only
# if self.story.site_url == 'blogtr' and self.chapter_name == '2':
# self.page_list.append("-0S2VlsSH424/UTqPDOXh7HI/AAAAAAAAAZg/P2gifiG1dEo")
threads = list()
for page_url in self.page_list:
page = Page(self, page_number=page_number, page_url=page_url)
thread = threading.Thread(target=page.page_download_get_link_only, args=(self.chapter_result,))
threads.append(thread)
page_number += 1
utils.threading_run(threads=threads)
self.validate_chapter()
if self.chapter_result['status']:
self.import_chapter_to_mongo()
return self.chapter_result
def upload_photos(self, album_title, list_of_files):
"""
:param album_title:
:param list_of_files: List of files to upload
:return:
"""
print "\nGoing to upload album {0}".format(album_title)
self.insert_album(album_title=album_title)
album = self.get_album(album_title)
threads = list()
for file_path in list_of_files:
threads.append(threading.Thread(target=self.upload_photo, args=(album, file_path)))
# self.upload_photo(album, file_path)
utils.threading_run(threads=threads, pool_size=5)
def chapter_download(self, move_chapter_to_img_dir=True):
"""
:param move_chapter_to_img_dir: Move downloaded chapter to image dir
:return: chapter_result
:type return: dict()
Validate page_list: if unable to get or is null => Return default chapter_result
Remove self.chapter_dir : Make sure all old files will be deleted
Append all pages to threading and run
Validate chapter after downloading
Move chapter to img_dir and import result to mongodb
"""
if self.page_list == list():
return self.chapter_result
utils.rmdir(self.chapter_dir)
utils.mkdir(self.chapter_dir)
page_number = int(self.start_page)
print "\nDOWNLOADING Story : {0} - chapter-{1} - site_url {2} - {3} pages"\
.format(self.story.story_name, self.chapter_name, self.story.site_url, len(self.page_list))
# Enable this for debug only
# if self.story.site_url == 'blogtr' and self.chapter_name == '2':
# self.page_list.append("-0S2VlsSH424/UTqPDOXh7HI/AAAAAAAAAZg/P2gifiG1dEo")
threads = list()
for page_url in self.page_list:
page = Page(self, page_number=page_number, page_url=page_url)
thread = threading.Thread(target=page.page_download, args=(self.chapter_result,))
threads.append(thread)
page_number += 1
utils.threading_run(threads=threads)
self.validate_chapter()
if self.chapter_result['status']:
if move_chapter_to_img_dir:
self.chapter_result['status'] = self.do_move_chapter_to_img_dir()
self.import_chapter_to_mongo()
return self.chapter_result
