def test_complete_download_from_scratch(self):
app_state = AppState()
dconf = DownloadConfiguration(number_of_images=10,
images_per_category=10,
download_destination=self.image_net_home)
app_state.set_configuration(dconf)
downloader = StatefulDownloader(app_state)
results = []
failed_urls = []
successful_urls = []
for result in downloader:
results.append(result)
failed_urls.extend(result.failed_urls)
successful_urls.extend(result.succeeded_urls)
self.assertEqual(failed_urls, [])
self.assertEqual(successful_urls,
['url1', 'url2', 'url3', 'url4', 'url5'])
self.assertEqual(downloader.progress_info.total_downloaded, 5)
self.assertEqual(downloader.progress_info.total_failed, 0)
self.assertTrue(downloader.progress_info.finished)
def test_stopping_and_resuming_with_new_instance(self):
app_state = AppState()
dconf = DownloadConfiguration(number_of_images=10,
images_per_category=12,
batch_size=3,
download_destination=self.image_net_home)
app_state.set_configuration(dconf)
downloader = StatefulDownloader(app_state)
for result in downloader:
break
app_state = AppState()
downloader = StatefulDownloader(app_state)
failed_urls = []
successful_urls = []
for result in downloader:
failed_urls.extend(result.failed_urls)
successful_urls.extend(result.succeeded_urls)
break
self.assertEqual(failed_urls, [])
self.assertEqual(successful_urls, ['url4', 'url5'])
self.assertEqual(downloader.progress_info.total_downloaded, 5)
self.assertEqual(downloader.progress_info.total_failed, 0)
self.assertTrue(downloader.progress_info.finished)
def test_remembers_number_of_images_downloaded_for_each_category(self):
app_state = AppState()
dconf = DownloadConfiguration(number_of_images=4,
images_per_category=1,
batch_size=2,
download_destination=self.image_net_home)
app_state.set_configuration(dconf)
downloader = StatefulDownloader(app_state)
for result in downloader:
break
app_state = AppState()
downloader = StatefulDownloader(app_state)
failed_urls = []
successful_urls = []
for result in downloader:
failed_urls.extend(result.failed_urls)
successful_urls.extend(result.succeeded_urls)
break
self.assertEqual(failed_urls, [])
self.assertEqual(successful_urls, ['url4', 'url5'])
self.assertEqual(downloader.progress_info.total_downloaded, 4)
self.assertEqual(downloader.progress_info.total_failed, 0)
self.assertTrue(downloader.progress_info.finished)
def test_start_pause_and_resume(self):
app_state = AppState()
manager = DownloadManager(app_state)
conf = DownloadConfiguration(number_of_images=5,
images_per_category=1,
download_destination=self.image_net_home)
app_state.set_configuration(conf)
paused_spy = QSignalSpy(manager.downloadPaused)
resumed_spy = QSignalSpy(manager.downloadResumed)
finished_spy = QSignalSpy(manager.allDownloaded)
manager.start()
manager.pause_download()
received = paused_spy.wait(timeout=500)
self.assertTrue(received)
time.sleep(0.5)
manager.resume_download()
received = finished_spy.wait(timeout=500)
self.assertTrue(received)
self._assert_expected_directories_exist()
self._assert_files_are_correct()
self.stop_the_thread(manager)
def test_creates_files_as_expected(self):
app_state = AppState()
dconf = DownloadConfiguration(number_of_images=4,
images_per_category=1,
batch_size=2,
download_destination=self.image_net_home)
app_state.set_configuration(dconf)
downloader = StatefulDownloader(app_state)
for result in downloader:
break
app_state = AppState()
downloader = StatefulDownloader(app_state)
for result in downloader:
pass
fnames = []
for dirname, dirs, file_names in os.walk(self.image_net_home):
fnames.extend(file_names)
expected_names = ['1', '2', '3', '4']
self.assertEqual(set(fnames), set(expected_names))
def test_progress_zero_by_zero(self):
app_state = AppState()
new_conf = DownloadConfiguration(number_of_images=0,
images_per_category=83,
download_destination='481516')
app_state.set_configuration(new_conf)
self.assertAlmostEqual(app_state.calculate_progress(), 0)
def test_errors_after_new_configuration(self):
app_state = AppState()
new_conf = DownloadConfiguration(number_of_images=0,
images_per_category=83,
download_destination='481516')
app_state.add_error('abc')
app_state.set_configuration(new_conf)
self.assertEqual(app_state.errors, [])
def test_inprogress_after_configuring(self):
app_state = AppState()
app_state.update_progress(result=Result([''], ['afef']))
app_state.mark_finished()
new_conf = DownloadConfiguration(number_of_images=0,
images_per_category=83,
download_destination='481516')
app_state.set_configuration(new_conf)
self.assertFalse(app_state.inprogress)
def test_folders_are_created(self):
app_state = AppState()
manager = DownloadManager(app_state)
conf = DownloadConfiguration(number_of_images=5,
images_per_category=10,
download_destination=self.image_net_home)
app_state.set_configuration(conf)
self.wait_for_completion(manager)
self._assert_expected_directories_exist()
self.stop_the_thread(manager)
def test_case_when_requested_number_of_images_is_greater_than_total(self):
app_state = AppState()
manager = DownloadManager(app_state)
conf = DownloadConfiguration(number_of_images=50,
images_per_category=100,
download_destination=self.image_net_home)
app_state.set_configuration(conf)
self.wait_for_completion(manager)
self._assert_files_are_correct()
self.stop_the_thread(manager)
def _assert_signal_emitted(self, signal):
app_state = AppState()
manager = DownloadManager(app_state)
conf = DownloadConfiguration(number_of_images=5,
images_per_category=10,
download_destination=self.image_net_home)
app_state.set_configuration(conf)
signal = getattr(manager, signal)
spy = QSignalSpy(signal)
manager.start()
received = spy.wait(timeout=500)
self.assertTrue(received)
self.stop_the_thread(manager)
def test_progress(self):
app_state = AppState()
new_conf = DownloadConfiguration(number_of_images=10,
images_per_category=83,
download_destination='481516')
last_result = Result(failed_urls=['1', 'one'], succeeded_urls=['x'])
progress_info = ProgressInfo(total_downloaded=9,
total_failed=38,
finished=False,
last_result=last_result)
app_state.set_configuration(new_conf)
app_state.set_progress_info(progress_info)
self.assertAlmostEqual(app_state.calculate_progress(), 0.9)
def test_reconfiguration(self):
app_state = AppState()
dconf = DownloadConfiguration(number_of_images=4,
images_per_category=1,
batch_size=2,
download_destination=self.image_net_home)
app_state.set_configuration(dconf)
downloader = StatefulDownloader(app_state)
for result in downloader:
break
shutil.rmtree(self.image_net_home)
os.makedirs(self.image_net_home)
app_state = AppState()
downloader = StatefulDownloader(app_state)
dconf = DownloadConfiguration(number_of_images=2,
images_per_category=2,
batch_size=2,
download_destination=self.image_net_home)
app_state.set_configuration(dconf)
failed_urls = []
successful_urls = []
for result in downloader:
failed_urls.extend(result.failed_urls)
successful_urls.extend(result.succeeded_urls)
break
self.assertEqual(successful_urls, ['url1', 'url2'])
self.assertEqual(downloader.progress_info.total_downloaded, 2)
self.assertEqual(downloader.progress_info.total_failed, 0)
fnames = []
for dirname, dirs, file_names in os.walk(self.image_net_home):
fnames.extend(file_names)
expected_names = ['1', '2']
self.assertEqual(set(fnames), set(expected_names))
def test_data_persistence(self):
app_state = AppState()
new_conf = DownloadConfiguration(number_of_images=9309,
images_per_category=83,
download_destination='481516')
last_result = Result(failed_urls=['1', 'one'], succeeded_urls=['x'])
progress_info = ProgressInfo(total_downloaded=192,
total_failed=38,
finished=False,
last_result=last_result)
position = Position(3, 1)
counts = {'wnid1': 29, 'wnid10': 3}
internal = InternalState(iterator_position=position,
category_counts=counts,
file_index=322)
app_state.set_configuration(new_conf)
app_state.set_progress_info(progress_info)
app_state.set_internal_state(internal)
app_state.save()
app_state = AppState()
conf = app_state.download_configuration
self.assertEqual(conf.download_destination, '481516')
self.assertEqual(conf.number_of_images, 9309)
self.assertEqual(conf.images_per_category, 83)
progress_info = app_state.progress_info
self.assertEqual(progress_info.total_downloaded, 192)
self.assertEqual(progress_info.total_failed, 38)
self.assertEqual(progress_info.finished, False)
self.assertEqual(progress_info.last_result.failed_urls, ['1', 'one'])
self.assertEqual(progress_info.last_result.succeeded_urls, ['x'])
internal = app_state.internal_state
self.assertEqual(internal.iterator_position.word_id_offset, 3)
self.assertEqual(internal.iterator_position.url_offset, 1)
self.assertEqual(internal.category_counts, counts)
self.assertEqual(internal.file_index, 322)
def test_images_per_category_argument(self):
app_state = AppState()
manager = DownloadManager(app_state)
conf = DownloadConfiguration(number_of_images=5,
images_per_category=1,
download_destination=self.image_net_home,
batch_size=1)
app_state.set_configuration(conf)
self.wait_for_completion(manager)
files_count = 0
for dirname, dirs, file_names in os.walk(self.image_net_home):
files_count += len(file_names)
self.assertEqual(files_count, 2)
self.stop_the_thread(manager)
def test_to_json(self):
app_state = AppState()
new_conf = DownloadConfiguration(number_of_images=9309,
images_per_category=83,
download_destination='481516')
last_result = Result(failed_urls=['1', 'one'], succeeded_urls=['x'])
progress_info = ProgressInfo(total_downloaded=192,
total_failed=38,
finished=False,
last_result=last_result)
position = Position(3, 1)
counts = {'wnid1': 29, 'wnid10': 3}
internal = InternalState(iterator_position=position,
category_counts=counts,
file_index=322)
app_state.set_configuration(new_conf)
app_state.set_progress_info(progress_info)
app_state.set_internal_state(internal)
app_state.add_error('Some error')
state_data = json.loads(app_state.to_json())
self.assertEqual(state_data['downloadPath'], '481516')
self.assertEqual(state_data['numberOfImages'], 9309)
self.assertEqual(state_data['imagesPerCategory'], 83)
self.assertNotEqual(state_data['timeLeft'], '')
self.assertEqual(state_data['imagesLoaded'], 192)
self.assertEqual(state_data['failures'], 38)
self.assertEqual(state_data['failedUrls'], ['1', 'one'])
self.assertEqual(state_data['succeededUrls'], ['x'])
self.assertEqual(state_data['errors'], ['Some error'])
self.assertAlmostEqual(state_data['progress'], 192.0 / 9309)
class StateManager(QtCore.QObject):
stateChanged = QtCore.pyqtSignal()
exceptionRaised = QtCore.pyqtSignal(str, arguments=['message'])
def __init__(self):
super().__init__()
self._app_state = AppState()
self._log_path = config.log_path
if self._app_state.progress_info.finished:
self._state = 'finished'
elif self._app_state.inprogress:
self._state = 'paused'
else:
self._state = 'initial'
self._reset_log()
self._strategy = self.get_strategy()
self._connect_signals()
def _connect_signals(self):
def handle_loaded(urls):
self.stateChanged.emit()
def handle_failed(urls):
self._log_failures(self._log_path, urls)
self.stateChanged.emit()
def handle_paused():
self._state = 'paused'
self.stateChanged.emit()
def handle_allDownloaded():
self._state = 'finished'
self._app_state.mark_finished()
self._app_state.save()
self.stateChanged.emit()
def handle_exception(message):
self._state = 'error'
self.exceptionRaised.emit(message)
self._strategy.imagesLoaded.connect(handle_loaded)
self._strategy.downloadFailed.connect(handle_failed)
self._strategy.downloadPaused.connect(handle_paused)
self._strategy.allDownloaded.connect(handle_allDownloaded)
self._strategy.exceptionRaised.connect(handle_exception)
def _reset_log(self):
if not os.path.exists(config.app_data_folder):
os.mkdir(config.app_data_folder)
if os.path.isfile(self._log_path):
os.remove(self._log_path)
with open(self._log_path, 'w') as f:
f.write('')
def _log_failures(self, log_path, urls):
with open(log_path, 'a') as f:
lines = '\n'.join(urls)
f.write(lines)
def get_strategy(self):
return DummyStrategy()
@QtCore.pyqtSlot()
def start_download(self):
if self._state == 'ready':
self._state = 'running'
self.stateChanged.emit()
self._strategy.start()
@QtCore.pyqtSlot(str, int, int)
def configure(self, destination, number_of_images,
images_per_category):
if self._state not in ['initial', 'ready']:
return
self._app_state.reset()
conf = DownloadConfiguration(number_of_images=number_of_images,
images_per_category=images_per_category,
download_destination=destination,
batch_size=config.default_batch_size)
if conf.is_valid:
self._state = 'ready'
path = self._parse_url(destination)
conf.download_destination = path
self._app_state.set_configuration(conf)
else:
self._state = 'initial'
self._generate_error_messages(conf)
self.stateChanged.emit()
def _generate_error_messages(self, download_conf):
for e in download_conf.errors:
self._app_state.add_error(e)
def _parse_url(self, file_uri):
p = urlparse(file_uri)
return os.path.abspath(os.path.join(p.netloc, p.path))
@QtCore.pyqtSlot()
def pause(self):
if self._state == 'running':
self._state = 'pausing'
self.stateChanged.emit()
self._strategy.pause_download()
@QtCore.pyqtSlot()
def resume(self):
if self._state == 'paused':
self._state = 'running'
self._strategy.resume_download()
self.stateChanged.emit()
@QtCore.pyqtSlot()
def reset(self):
if self._state not in ['running', 'pausing']:
self._state = 'initial'
self._reset_log()
self._app_state.reset()
self._strategy.quit()
self._strategy = self.get_strategy()
self._connect_signals()
self._app_state.save()
self.stateChanged.emit()
@QtCore.pyqtProperty(str)
def download_state(self):
return self._state
@QtCore.pyqtProperty(str)
def state_data_json(self):
return self._app_state.to_json()
@QtCore.pyqtProperty(str)
def time_remaining(self):
return self._app_state.time_remaining