def test_change_config(self):
downloader = Downloader(Config(DEFAULT_INI_PATH))
customize_dict = {
'multi': {
'process_number': 1,
'thread_number': 1,
'delay': 2.0
},
'customize': {
'use': 1,
'char': 'a'
}
}
customize_ini_path = './tests/config/customize_config.ini'
config = Config(customize_ini_path, customize_dict)
downloader.change_config(config)
def test_chinese_support(self):
test_gb2312_urls = ["https://www.biqukan.com/50_50758/"]
downloader = Downloader(Config(DEFAULT_INI_PATH))
downloader.enable_chinese_transcoding()
result = downloader.get_result(test_gb2312_urls)
downloader.disable_chinese_transcoding()
assert len(result.get_urls_detail_dict()) == 1
def test_typical_urls(self, test_urls, expect_failed_urls_number,
expect_finished_urls_number):
downloader = Downloader(Config(DEFAULT_INI_PATH))
self.fast_download(downloader)
result = downloader.get_result(test_urls)
result.show_time_cost()
assert len(result.get_failed_urls()) == expect_failed_urls_number
assert len(result.get_finished_urls()) == expect_finished_urls_number
def test_customize_url_manager(self, test_urls, expect_failed_urls_number,
expect_finished_urls_number):
url_manger = UrlManager()
for url in test_urls:
url_manger.add_url(url)
downloader = Downloader(Config(DEFAULT_INI_PATH))
self.fast_download(downloader)
result = downloader.get_result(test_urls, url_manger)
assert len(result.get_failed_urls()) == expect_failed_urls_number
assert len(result.get_finished_urls()) == expect_finished_urls_number
def start(self):
t1 = time.time()
downloader = Downloader(
Config("hardworking_av_studio.ini", HARDWORKING_CONFIG_DICT,
HARDWORKING_CONFIG_SCHEMA))
urls = ['https://www.dmmsee.zone/studio/0']
urls.extend([
'https://www.dmmsee.zone/studio/{}{}'.format(i, word)
for i in range(1, 40) for word in ' ' + string.ascii_lowercase
])
urls.extend([
'https://www.dmmsee.zone/studio/{}'.format(i)
for i in range(40, 400)
])
print(" config ".center(shutil.get_terminal_size().columns - 1, '*'))
downloader.config.list_config()
print(" download urls ".center(shutil.get_terminal_size().columns - 1,
'*'))
self.result = downloader.get_result(urls)
self.result.show_time_cost()
self.result.show_urls_status()
print(" retry failed urls ".center(
shutil.get_terminal_size().columns - 1, '*'))
self.result.retry_failed_urls()
self.result.show_urls_status()
if os.path.exists("hardworking_av_studio.txt"):
os.remove("hardworking_av_studio.txt")
print(" analyzing result ".center(
shutil.get_terminal_size().columns - 1, '*'))
tmp_time = time.time()
analyzing_result_process_number = downloader.config.get_config(
'multi', 'analyzing_result_process_number')
queue = SimpleQueue()
for i in range(analyzing_result_process_number):
Process(target=self.screen_by_mini_date,
args=(self.result.get_finished_urls()
[i::analyzing_result_process_number],
queue)).start()
for i in tqdm(range(len(self.result.get_finished_urls())),
total=len(self.result.get_finished_urls()),
desc="analyzing result",
unit="result",
postfix={"process": analyzing_result_process_number}):
queue.get()
print("\nanalysis completed... time cost {:.2f}s".format(time.time() -
tmp_time))
print(" result ".center(shutil.get_terminal_size().columns, '*'))
print("The result has been written to the current folder:",
os.path.join(os.getcwd(), "hardworking_av_studio.txt"))
print("total time cost {:.2f}s".format(time.time() - t1))
return True
def setup_method():
if os.path.exists(CONFIG_DIR_PATH):
rmtree(CONFIG_DIR_PATH)
os.mkdir(CONFIG_DIR_PATH)
Config.make_default_ini(DEFAULT_INI_PATH)
def __init__(self):
self.novel_name = None
self.downloader = Downloader(
Config("novel_downloader.ini", NOVEL_DOWNLOADER_CONFIG_DICT))
self.downloader.enable_chinese_transcoding()
def test_get_result_empty_urls(self):
downloader = Downloader(Config(DEFAULT_INI_PATH))
assert downloader.get_result(test_empty_urls) is None