async def test_failed_backup(time, ha: HaSource, supervisor: SimulatedSupervisor, config: Config, interceptor: RequestInterceptor): # create a blocking backup interceptor.setError(URL_MATCH_BACKUP_FULL, 524) config.override(Setting.NEW_BACKUP_TIMEOUT_SECONDS, 0) await supervisor.toggleBlockBackup() backup_immediate = await ha.create(CreateOptions(time.now(), "Some Name")) assert isinstance(backup_immediate, PendingBackup) assert backup_immediate.name() == "Some Name" assert not ha.check() assert not backup_immediate.isFailed() await supervisor.toggleBlockBackup() # let the backup attempt to complete await asyncio.wait({ha._pending_backup_task}) # verify it failed with the expected http error assert backup_immediate.isFailed() assert backup_immediate._exception.status == 524 backups = list((await ha.get()).values()) assert len(backups) == 1 assert backups[0] is backup_immediate # verify we can create a new backup immediately interceptor.clear() await ha.create(CreateOptions(time.now(), "Some Name")) assert len(await ha.get()) == 1
async def test_upload_resume(drive: DriveSource, time, snapshot_helper: SnapshotHelper, google: SimulatedGoogle, interceptor: RequestInterceptor): from_snapshot, data = await snapshot_helper.createFile() interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, attempts=1, status=500) # Upload, which will fail with pytest.raises(GoogleInternalError): await drive.save(from_snapshot, data) # Verify we uploaded one chunk assert google.chunks == [BASE_CHUNK_SIZE] # Retry the upload, which shoudl now pass interceptor.clear() data.position(0) drive_snapshot = await drive.save(from_snapshot, data) from_snapshot.addSource(drive_snapshot) assert google.chunks == [ BASE_CHUNK_SIZE, BASE_CHUNK_SIZE, (data.size()) - BASE_CHUNK_SIZE * 2 ] # Verify the data is correct data.position(0) await compareStreams(data, await drive.read(from_snapshot))
async def test_resume_session_abandoned_on_http4XX( time, drive: DriveSource, config: Config, server, snapshot_helper, interceptor: RequestInterceptor): from_snapshot, data = await snapshot_helper.createFile() # Configure the upload to fail after the first upload chunk interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 402, 1) with pytest.raises(ClientResponseError): await drive.save(from_snapshot, data) # Verify a requst was made to start the upload but not cached assert server.wasUrlRequested( "/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true") assert drive.drivebackend.last_attempt_count == 1 assert drive.drivebackend.last_attempt_location is None assert drive.drivebackend.last_attempt_metadata is None # upload again, which should retry server.urls.clear() interceptor.clear() data.position(0) snapshot = await drive.save(from_snapshot, data) assert server.wasUrlRequested(URL_START_UPLOAD) # Verify the uploaded bytes are identical from_snapshot.addSource(snapshot) download = await drive.read(from_snapshot) data.position(0) await compareStreams(data, download)
async def test_resume_session_abandoned_after_a_long_time( time: FakeTime, drive: DriveSource, config: Config, server: SimulationServer, snapshot_helper, interceptor: RequestInterceptor): from_snapshot, data = await snapshot_helper.createFile() # Configure the upload to fail after the first upload chunk interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1) with pytest.raises(ClientResponseError): await drive.save(from_snapshot, data) # Verify it reuses the session a few times assert server.wasUrlRequested(URL_START_UPLOAD) assert drive.drivebackend.last_attempt_count == 1 assert drive.drivebackend.last_attempt_location is not None assert drive.drivebackend.last_attempt_metadata is not None data.position(0) with pytest.raises(ClientResponseError): await drive.save(from_snapshot, data) assert drive.drivebackend.last_attempt_count == 2 assert drive.drivebackend.last_attempt_location is not None assert drive.drivebackend.last_attempt_metadata is not None last_location = drive.drivebackend.last_attempt_location # Fast forward a lot, then verify the session is restarted server.urls.clear() interceptor.clear() time.advance(duration=UPLOAD_SESSION_EXPIRATION_DURATION) data.position(0) await drive.save(from_snapshot, data) assert interceptor.urlWasCalled(URL_START_UPLOAD) assert not interceptor.urlWasCalled(last_location)
async def test_failed_snapshot(time, ha: HaSource, supervisor: SimulatedSupervisor, config: Config, interceptor: RequestInterceptor): # create a blocking snapshot interceptor.setError(URL_MATCH_SNAPSHOT_FULL, 524) config.override(Setting.NEW_SNAPSHOT_TIMEOUT_SECONDS, 0) await supervisor.toggleBlockSnapshot() snapshot_immediate = await ha.create(CreateOptions(time.now(), "Some Name")) assert isinstance(snapshot_immediate, PendingSnapshot) assert snapshot_immediate.name() == "Some Name" assert not ha.check() assert not snapshot_immediate.isFailed() await supervisor.toggleBlockSnapshot() # let the snapshot attempt to complete await asyncio.wait({ha._pending_snapshot_task}) # verify it failed with the expected http error assert snapshot_immediate.isFailed() assert snapshot_immediate._exception.status == 524 snapshots = list((await ha.get()).values()) assert len(snapshots) == 1 assert snapshots[0] is snapshot_immediate # verify we can create a new snapshot immediately interceptor.clear() await ha.create(CreateOptions(time.now(), "Some Name")) assert len(await ha.get()) == 1
async def test_chunk_upload_resets_attempt_counter(time: FakeTime, drive: DriveSource, config: Config, server: SimulationServer, backup_helper: BackupHelper, interceptor: RequestInterceptor): from_backup, data = await backup_helper.createFile(size=1024 * 1024 * 10) # Configure the upload to fail after the first upload chunk interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1) with pytest.raises(ClientResponseError): await drive.save(from_backup, data) data.position(0) with pytest.raises(ClientResponseError): await drive.save(from_backup, data) # Verify the session was started assert interceptor.urlWasCalled(URL_START_UPLOAD) assert interceptor.urlWasCalled(URL_MATCH_UPLOAD_PROGRESS) assert drive.drivebackend.last_attempt_count == 2 location = drive.drivebackend.last_attempt_location assert location is not None # Allow one more chunk to succeed interceptor.clear() interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 2) data.position(0) with pytest.raises(ClientResponseError): await drive.save(from_backup, data) # Verify the session was reused and the attempt counter was reset assert not interceptor.urlWasCalled(URL_START_UPLOAD) assert interceptor.urlWasCalled(URL_MATCH_UPLOAD_PROGRESS) assert interceptor.urlWasCalled(URL(location).path) assert drive.drivebackend.last_attempt_count == 1 assert drive.drivebackend.last_attempt_location == location
async def test_failure_logging(updater: HaUpdater, server, time: FakeTime, interceptor: RequestInterceptor): interceptor.setError(URL_MATCH_CORE_API, 501) assert getLast() is None await updater.update() assert getLast() is None time.advance(minutes=1) await updater.update() assert getLast() is None time.advance(minutes=5) await updater.update() assert getLast().msg == REASSURING_MESSAGE.format(501) last_log = getLast() time.advance(minutes=5) await updater.update() assert getLast() is not last_log assert getLast().msg == REASSURING_MESSAGE.format(501) last_log = getLast() interceptor.clear() await updater.update() assert getLast() is last_log
async def test_failure_backoff_other(updater: HaUpdater, server, time: FakeTime, interceptor: RequestInterceptor): interceptor.setError(URL_MATCH_CORE_API, 400) for x in range(9): await updater.update() assert time.sleeps == [60, 120, 240, 300, 300, 300, 300, 300, 300] interceptor.clear() await updater.update() assert time.sleeps == [60, 120, 240, 300, 300, 300, 300, 300, 300]
async def test_supervisor_permission_error(time, ha: HaSource, interceptor: RequestInterceptor, global_info: GlobalInfo): interceptor.setError(URL_MATCH_MISC_INFO, 403) with pytest.raises(SupervisorPermissionError): await ha.init() interceptor.clear() interceptor.setError(URL_MATCH_MISC_INFO, 404) with pytest.raises(ClientResponseError): await ha.init()
async def test_delete_error(time, ha: HaSource, interceptor: RequestInterceptor): snapshot = await ha.create(CreateOptions(time.now(), "Some Name")) full = DummySnapshot(snapshot.name(), snapshot.date(), snapshot.size(), snapshot.slug(), "dummy") full.addSource(snapshot) interceptor.setError(URL_MATCH_SNAPSHOT_DELETE, 400) with pytest.raises(HomeAssistantDeleteError): await ha.delete(full) interceptor.clear() await ha.delete(full)
async def test_google_internal_error(drive, server, time: FakeTime, interceptor: RequestInterceptor): interceptor.setError(URL_MATCH_FILE, 500) with pytest.raises(GoogleInternalError): await drive.get() assert time.sleeps == RETRY_EXHAUSTION_SLEEPS time.clearSleeps() interceptor.clear() interceptor.setError(URL_MATCH_FILE, 500) with pytest.raises(GoogleInternalError): await drive.get() assert time.sleeps == RETRY_EXHAUSTION_SLEEPS
async def test_delete_error(time, ha: HaSource, interceptor: RequestInterceptor): backup = await ha.create(CreateOptions(time.now(), "Some Name")) full = DummyBackup(backup.name(), backup.date(), backup.size(), backup.slug(), "dummy") full.addSource(backup) interceptor.setError(URL_MATCH_BACKUP_DELETE, 400) with pytest.raises(HomeAssistantDeleteError): await ha.delete(full) interceptor.clear() await ha.delete(full)
async def test_upgrade_all_config(ha: HaSource, supervisor: SimulatedSupervisor, interceptor: RequestInterceptor, config: Config, server_url): """Verify that converting all upgradeable config optiosn works as expected""" # overwrite the addon options with old values supervisor._options = { Setting.DEPRECTAED_MAX_BACKUPS_IN_HA.value: 1, Setting.DEPRECTAED_MAX_BACKUPS_IN_GOOGLE_DRIVE.value: 2, Setting.DEPRECATED_DAYS_BETWEEN_BACKUPS.value: 5, Setting.DEPRECTAED_IGNORE_OTHER_BACKUPS.value: True, Setting.DEPRECTAED_IGNORE_UPGRADE_BACKUPS.value: True, Setting.DEPRECTAED_BACKUP_TIME_OF_DAY.value: "01:11", Setting.DEPRECTAED_DELETE_BEFORE_NEW_BACKUP.value: True, Setting.DEPRECTAED_BACKUP_NAME.value: "test", Setting.DEPRECTAED_SPECIFY_BACKUP_FOLDER.value: True, Setting.DEPRECTAED_NOTIFY_FOR_STALE_BACKUPS.value: False, Setting.DEPRECTAED_ENABLE_BACKUP_STALE_SENSOR.value: False, Setting.DEPRECTAED_ENABLE_BACKUP_STATE_SENSOR.value: False, Setting.DEPRECATED_BACKUP_PASSWORD.value: "test password", } await ha.init() assert not config.mustSaveUpgradeChanges() assert interceptor.urlWasCalled(URL_MATCH_SELF_OPTIONS) # Verify the config was upgraded assert supervisor._options == { Setting.MAX_BACKUPS_IN_HA.value: 1, Setting.MAX_BACKUPS_IN_GOOGLE_DRIVE.value: 2, Setting.DAYS_BETWEEN_BACKUPS.value: 5, Setting.IGNORE_OTHER_BACKUPS.value: True, Setting.IGNORE_UPGRADE_BACKUPS.value: True, Setting.BACKUP_TIME_OF_DAY.value: "01:11", Setting.DELETE_BEFORE_NEW_BACKUP.value: True, Setting.BACKUP_NAME.value: "test", Setting.SPECIFY_BACKUP_FOLDER.value: True, Setting.NOTIFY_FOR_STALE_BACKUPS.value: False, Setting.ENABLE_BACKUP_STALE_SENSOR.value: False, Setting.ENABLE_BACKUP_STATE_SENSOR.value: False, Setting.BACKUP_PASSWORD.value: "test password", Setting.CALL_BACKUP_SNAPSHOT.value: True, } interceptor.clear() await ha.init() assert not interceptor.urlWasCalled(URL_MATCH_SELF_OPTIONS)
async def verify_upload_resumed(time, drive: DriveSource, config: Config, server: SimulationServer, interceptor: RequestInterceptor, status, snapshot_helper, expected=ClientResponseError): from_snapshot, data = await snapshot_helper.createFile() # Configure the upload to fail after the first upload chunk interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, status, 1) with pytest.raises(expected): await drive.save(from_snapshot, data) # Verify a requst was made to start the upload assert server.wasUrlRequested( "/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true") assert drive.drivebackend.last_attempt_location is not None assert drive.drivebackend.last_attempt_metadata is not None last_location = drive.drivebackend.last_attempt_location # Retry the upload and let is succeed server.urls.clear() interceptor.clear() data.position(0) snapshot = await drive.save(from_snapshot, data) # We shoudl nto see the upload "initialize" url assert not server.wasUrlRequested( "/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true") # We should see the last location url (which has a unique token) reused to resume the upload assert server.wasUrlRequested(last_location) # The saved metadata should be cleared out. assert drive.drivebackend.last_attempt_count == 1 assert drive.drivebackend.last_attempt_location is None assert drive.drivebackend.last_attempt_metadata is None # Verify the uploaded bytes are identical from_snapshot.addSource(snapshot) download = await drive.read(from_snapshot) data.position(0) await compareStreams(data, download)
async def test_resume_session_reused_on_http410(time, drive: DriveSource, config: Config, server: SimulationServer, backup_helper: BackupHelper, interceptor: RequestInterceptor): from_backup, data = await backup_helper.createFile() # Configure the upload to fail interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 500) with pytest.raises(GoogleInternalError): await drive.save(from_backup, data) # Verify a requst was made to start the upload assert server.wasUrlRequested(URL_START_UPLOAD) assert drive.drivebackend.last_attempt_location is not None server.urls.clear() interceptor.clear() data.position(0) interceptor.setError(drive.drivebackend.last_attempt_location, 0, 410) await drive.save(from_backup, data)
async def test_resume_upload_attempts_exhausted( drive: DriveSource, time, snapshot_helper, interceptor: RequestInterceptor, google: SimulatedGoogle): # Allow an upload to update one chunk and then fail. from_snapshot, data = await snapshot_helper.createFile() interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, attempts=1, status=500) with pytest.raises(GoogleInternalError): await drive.save(from_snapshot, data) assert google.chunks == [BASE_CHUNK_SIZE] # Verify we have a cached location assert drive.drivebackend.last_attempt_location is not None assert drive.drivebackend.last_attempt_count == 1 last_location = drive.drivebackend.last_attempt_location for x in range(1, RETRY_SESSION_ATTEMPTS): data.position(0) with pytest.raises(GoogleInternalError): await drive.save(from_snapshot, data) assert drive.drivebackend.last_attempt_count == x + 1 # We should still be using the same location url assert drive.drivebackend.last_attempt_location == last_location # Another attempt should use another location url with pytest.raises(GoogleInternalError): data.position(0) await drive.save(from_snapshot, data) assert drive.drivebackend.last_attempt_count == 0 assert drive.drivebackend.last_attempt_location is not None assert drive.drivebackend.last_attempt_location != last_location # Now let it succeed interceptor.clear() data.position(0) drive_snapshot = await drive.save(from_snapshot, data) from_snapshot.addSource(drive_snapshot) # And verify the bytes are correct data.position(0) await compareStreams(data, await drive.read(from_snapshot))
async def test_resume_session_reused_on_http408( time, drive: DriveSource, config: Config, server: SimulationServer, snapshot_helper: SnapshotHelper, interceptor: RequestInterceptor): from_snapshot, data = await snapshot_helper.createFile() # Configure the upload to fail interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 408) with pytest.raises(GoogleTimeoutError): await drive.save(from_snapshot, data) # Verify a requst was made to start the upload assert server.wasUrlRequested(URL_START_UPLOAD) location = drive.drivebackend.last_attempt_location assert location is not None server.urls.clear() interceptor.clear() data.position(0) await drive.save(from_snapshot, data) assert interceptor.urlWasCalled(URL(location).path)
async def test_resume_session_reused_on_http410( time, drive: DriveSource, config: Config, server: SimulationServer, snapshot_helper: SnapshotHelper, interceptor: RequestInterceptor): from_snapshot, data = await snapshot_helper.createFile() # Configure the upload to fail interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 500) with pytest.raises(GoogleInternalError): await drive.save(from_snapshot, data) # Verify a requst was made to start the upload assert server.wasUrlRequested( "/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true") assert drive.drivebackend.last_attempt_location is not None server.urls.clear() interceptor.clear() data.position(0) interceptor.setError(drive.drivebackend.last_attempt_location, 0, 410) await drive.save(from_snapshot, data)
async def test_resume_session_reused_abonded_after_retries( time, drive: DriveSource, config: Config, server, snapshot_helper, interceptor: RequestInterceptor): from_snapshot, data = await snapshot_helper.createFile() # Configure the upload to fail after the first upload chunk interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1) with pytest.raises(ClientResponseError): await drive.save(from_snapshot, data) # Verify a requst was made to start the upload but not cached assert server.wasUrlRequested( "/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true") assert drive.drivebackend.last_attempt_count == 0 assert drive.drivebackend.last_attempt_location is not None assert drive.drivebackend.last_attempt_metadata is not None last_location = drive.drivebackend.last_attempt_location for x in range(1, RETRY_SESSION_ATTEMPTS + 1): server.urls.clear() interceptor.clear() interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501) data.position(0) with pytest.raises(ClientResponseError): await drive.save(from_snapshot, data) assert not server.wasUrlRequested( "/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true" ) assert server.wasUrlRequested(last_location) assert drive.drivebackend.last_attempt_count == x assert drive.drivebackend.last_attempt_location is last_location assert drive.drivebackend.last_attempt_metadata is not None # Next attempt should give up and restart the upload server.urls.clear() interceptor.clear() interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1) data.position(0) with pytest.raises(ClientResponseError): await drive.save(from_snapshot, data) assert server.wasUrlRequested( "/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true") assert not server.wasUrlRequested(last_location) assert drive.drivebackend.last_attempt_count == 0 # upload again, which should retry server.urls.clear() interceptor.clear() data.position(0) snapshot = await drive.save(from_snapshot, data) assert not server.wasUrlRequested( "/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true") # Verify the uploaded bytes are identical from_snapshot.addSource(snapshot) download = await drive.read(from_snapshot) data.position(0) await compareStreams(data, download)
async def test_resume_session_reused_abonded_after_retries(time, drive: DriveSource, config: Config, server: SimulationServer, backup_helper, interceptor: RequestInterceptor): from_backup, data = await backup_helper.createFile() # Configure the upload to fail after the first upload chunk interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1) with pytest.raises(ClientResponseError): await drive.save(from_backup, data) # Verify a requst was made to start the upload but not cached assert server.wasUrlRequested(URL_START_UPLOAD) assert drive.drivebackend.last_attempt_count == 1 assert drive.drivebackend.last_attempt_location is not None assert drive.drivebackend.last_attempt_metadata is not None last_location = drive.drivebackend.last_attempt_location for x in range(1, RETRY_SESSION_ATTEMPTS): server.urls.clear() interceptor.clear() interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501) data.position(0) with pytest.raises(ClientResponseError): await drive.save(from_backup, data) assert not server.wasUrlRequested(URL_START_UPLOAD) assert server.wasUrlRequested(last_location) assert drive.drivebackend.last_attempt_count == x + 1 assert drive.drivebackend.last_attempt_location is last_location assert drive.drivebackend.last_attempt_metadata is not None # Next attempt should give up and restart the upload server.urls.clear() interceptor.clear() interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1) data.position(0) with pytest.raises(ClientResponseError): await drive.save(from_backup, data) assert server.wasUrlRequested(URL_START_UPLOAD) assert not server.wasUrlRequested(last_location) assert drive.drivebackend.last_attempt_count == 1 # upload again, which should retry server.urls.clear() interceptor.clear() data.position(0) backup = await drive.save(from_backup, data) assert not server.wasUrlRequested(URL_START_UPLOAD) # Verify the uploaded bytes are identical from_backup.addSource(backup) download = await drive.read(from_backup) data.position(0) await compareStreams(data, download)