async def test_failed_backup(time, ha: HaSource,
supervisor: SimulatedSupervisor, config: Config,
interceptor: RequestInterceptor):
# create a blocking backup
interceptor.setError(URL_MATCH_BACKUP_FULL, 524)
config.override(Setting.NEW_BACKUP_TIMEOUT_SECONDS, 0)
await supervisor.toggleBlockBackup()
backup_immediate = await ha.create(CreateOptions(time.now(), "Some Name"))
assert isinstance(backup_immediate, PendingBackup)
assert backup_immediate.name() == "Some Name"
assert not ha.check()
assert not backup_immediate.isFailed()
await supervisor.toggleBlockBackup()
# let the backup attempt to complete
await asyncio.wait({ha._pending_backup_task})
# verify it failed with the expected http error
assert backup_immediate.isFailed()
assert backup_immediate._exception.status == 524
backups = list((await ha.get()).values())
assert len(backups) == 1
assert backups[0] is backup_immediate
# verify we can create a new backup immediately
interceptor.clear()
await ha.create(CreateOptions(time.now(), "Some Name"))
assert len(await ha.get()) == 1
async def test_upload_resume(drive: DriveSource, time,
snapshot_helper: SnapshotHelper,
google: SimulatedGoogle,
interceptor: RequestInterceptor):
from_snapshot, data = await snapshot_helper.createFile()
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, attempts=1, status=500)
# Upload, which will fail
with pytest.raises(GoogleInternalError):
await drive.save(from_snapshot, data)
# Verify we uploaded one chunk
assert google.chunks == [BASE_CHUNK_SIZE]
# Retry the upload, which shoudl now pass
interceptor.clear()
data.position(0)
drive_snapshot = await drive.save(from_snapshot, data)
from_snapshot.addSource(drive_snapshot)
assert google.chunks == [
BASE_CHUNK_SIZE, BASE_CHUNK_SIZE, (data.size()) - BASE_CHUNK_SIZE * 2
]
# Verify the data is correct
data.position(0)
await compareStreams(data, await drive.read(from_snapshot))
async def test_resume_session_abandoned_on_http4XX(
time, drive: DriveSource, config: Config, server, snapshot_helper,
interceptor: RequestInterceptor):
from_snapshot, data = await snapshot_helper.createFile()
# Configure the upload to fail after the first upload chunk
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 402, 1)
with pytest.raises(ClientResponseError):
await drive.save(from_snapshot, data)
# Verify a requst was made to start the upload but not cached
assert server.wasUrlRequested(
"/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true")
assert drive.drivebackend.last_attempt_count == 1
assert drive.drivebackend.last_attempt_location is None
assert drive.drivebackend.last_attempt_metadata is None
# upload again, which should retry
server.urls.clear()
interceptor.clear()
data.position(0)
snapshot = await drive.save(from_snapshot, data)
assert server.wasUrlRequested(URL_START_UPLOAD)
# Verify the uploaded bytes are identical
from_snapshot.addSource(snapshot)
download = await drive.read(from_snapshot)
data.position(0)
await compareStreams(data, download)
async def test_resume_session_abandoned_after_a_long_time(
time: FakeTime, drive: DriveSource, config: Config,
server: SimulationServer, snapshot_helper,
interceptor: RequestInterceptor):
from_snapshot, data = await snapshot_helper.createFile()
# Configure the upload to fail after the first upload chunk
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1)
with pytest.raises(ClientResponseError):
await drive.save(from_snapshot, data)
# Verify it reuses the session a few times
assert server.wasUrlRequested(URL_START_UPLOAD)
assert drive.drivebackend.last_attempt_count == 1
assert drive.drivebackend.last_attempt_location is not None
assert drive.drivebackend.last_attempt_metadata is not None
data.position(0)
with pytest.raises(ClientResponseError):
await drive.save(from_snapshot, data)
assert drive.drivebackend.last_attempt_count == 2
assert drive.drivebackend.last_attempt_location is not None
assert drive.drivebackend.last_attempt_metadata is not None
last_location = drive.drivebackend.last_attempt_location
# Fast forward a lot, then verify the session is restarted
server.urls.clear()
interceptor.clear()
time.advance(duration=UPLOAD_SESSION_EXPIRATION_DURATION)
data.position(0)
await drive.save(from_snapshot, data)
assert interceptor.urlWasCalled(URL_START_UPLOAD)
assert not interceptor.urlWasCalled(last_location)
async def test_failed_snapshot(time, ha: HaSource,
supervisor: SimulatedSupervisor, config: Config,
interceptor: RequestInterceptor):
# create a blocking snapshot
interceptor.setError(URL_MATCH_SNAPSHOT_FULL, 524)
config.override(Setting.NEW_SNAPSHOT_TIMEOUT_SECONDS, 0)
await supervisor.toggleBlockSnapshot()
snapshot_immediate = await ha.create(CreateOptions(time.now(),
"Some Name"))
assert isinstance(snapshot_immediate, PendingSnapshot)
assert snapshot_immediate.name() == "Some Name"
assert not ha.check()
assert not snapshot_immediate.isFailed()
await supervisor.toggleBlockSnapshot()
# let the snapshot attempt to complete
await asyncio.wait({ha._pending_snapshot_task})
# verify it failed with the expected http error
assert snapshot_immediate.isFailed()
assert snapshot_immediate._exception.status == 524
snapshots = list((await ha.get()).values())
assert len(snapshots) == 1
assert snapshots[0] is snapshot_immediate
# verify we can create a new snapshot immediately
interceptor.clear()
await ha.create(CreateOptions(time.now(), "Some Name"))
assert len(await ha.get()) == 1
async def test_chunk_upload_resets_attempt_counter(time: FakeTime, drive: DriveSource, config: Config, server: SimulationServer, backup_helper: BackupHelper, interceptor: RequestInterceptor):
from_backup, data = await backup_helper.createFile(size=1024 * 1024 * 10)
# Configure the upload to fail after the first upload chunk
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1)
with pytest.raises(ClientResponseError):
await drive.save(from_backup, data)
data.position(0)
with pytest.raises(ClientResponseError):
await drive.save(from_backup, data)
# Verify the session was started
assert interceptor.urlWasCalled(URL_START_UPLOAD)
assert interceptor.urlWasCalled(URL_MATCH_UPLOAD_PROGRESS)
assert drive.drivebackend.last_attempt_count == 2
location = drive.drivebackend.last_attempt_location
assert location is not None
# Allow one more chunk to succeed
interceptor.clear()
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 2)
data.position(0)
with pytest.raises(ClientResponseError):
await drive.save(from_backup, data)
# Verify the session was reused and the attempt counter was reset
assert not interceptor.urlWasCalled(URL_START_UPLOAD)
assert interceptor.urlWasCalled(URL_MATCH_UPLOAD_PROGRESS)
assert interceptor.urlWasCalled(URL(location).path)
assert drive.drivebackend.last_attempt_count == 1
assert drive.drivebackend.last_attempt_location == location
async def test_failure_logging(updater: HaUpdater, server, time: FakeTime,
interceptor: RequestInterceptor):
interceptor.setError(URL_MATCH_CORE_API, 501)
assert getLast() is None
await updater.update()
assert getLast() is None
time.advance(minutes=1)
await updater.update()
assert getLast() is None
time.advance(minutes=5)
await updater.update()
assert getLast().msg == REASSURING_MESSAGE.format(501)
last_log = getLast()
time.advance(minutes=5)
await updater.update()
assert getLast() is not last_log
assert getLast().msg == REASSURING_MESSAGE.format(501)
last_log = getLast()
interceptor.clear()
await updater.update()
assert getLast() is last_log
async def test_failure_backoff_other(updater: HaUpdater, server, time: FakeTime, interceptor: RequestInterceptor):
interceptor.setError(URL_MATCH_CORE_API, 400)
for x in range(9):
await updater.update()
assert time.sleeps == [60, 120, 240, 300, 300, 300, 300, 300, 300]
interceptor.clear()
await updater.update()
assert time.sleeps == [60, 120, 240, 300, 300, 300, 300, 300, 300]
async def test_supervisor_permission_error(time, ha: HaSource,
interceptor: RequestInterceptor,
global_info: GlobalInfo):
interceptor.setError(URL_MATCH_MISC_INFO, 403)
with pytest.raises(SupervisorPermissionError):
await ha.init()
interceptor.clear()
interceptor.setError(URL_MATCH_MISC_INFO, 404)
with pytest.raises(ClientResponseError):
await ha.init()
async def test_delete_error(time, ha: HaSource,
interceptor: RequestInterceptor):
snapshot = await ha.create(CreateOptions(time.now(), "Some Name"))
full = DummySnapshot(snapshot.name(), snapshot.date(), snapshot.size(),
snapshot.slug(), "dummy")
full.addSource(snapshot)
interceptor.setError(URL_MATCH_SNAPSHOT_DELETE, 400)
with pytest.raises(HomeAssistantDeleteError):
await ha.delete(full)
interceptor.clear()
await ha.delete(full)
async def test_google_internal_error(drive, server, time: FakeTime, interceptor: RequestInterceptor):
interceptor.setError(URL_MATCH_FILE, 500)
with pytest.raises(GoogleInternalError):
await drive.get()
assert time.sleeps == RETRY_EXHAUSTION_SLEEPS
time.clearSleeps()
interceptor.clear()
interceptor.setError(URL_MATCH_FILE, 500)
with pytest.raises(GoogleInternalError):
await drive.get()
assert time.sleeps == RETRY_EXHAUSTION_SLEEPS
async def test_delete_error(time, ha: HaSource,
interceptor: RequestInterceptor):
backup = await ha.create(CreateOptions(time.now(), "Some Name"))
full = DummyBackup(backup.name(), backup.date(), backup.size(),
backup.slug(), "dummy")
full.addSource(backup)
interceptor.setError(URL_MATCH_BACKUP_DELETE, 400)
with pytest.raises(HomeAssistantDeleteError):
await ha.delete(full)
interceptor.clear()
await ha.delete(full)
async def test_upgrade_all_config(ha: HaSource,
supervisor: SimulatedSupervisor,
interceptor: RequestInterceptor,
config: Config, server_url):
"""Verify that converting all upgradeable config optiosn works as expected"""
# overwrite the addon options with old values
supervisor._options = {
Setting.DEPRECTAED_MAX_BACKUPS_IN_HA.value: 1,
Setting.DEPRECTAED_MAX_BACKUPS_IN_GOOGLE_DRIVE.value: 2,
Setting.DEPRECATED_DAYS_BETWEEN_BACKUPS.value: 5,
Setting.DEPRECTAED_IGNORE_OTHER_BACKUPS.value: True,
Setting.DEPRECTAED_IGNORE_UPGRADE_BACKUPS.value: True,
Setting.DEPRECTAED_BACKUP_TIME_OF_DAY.value: "01:11",
Setting.DEPRECTAED_DELETE_BEFORE_NEW_BACKUP.value: True,
Setting.DEPRECTAED_BACKUP_NAME.value: "test",
Setting.DEPRECTAED_SPECIFY_BACKUP_FOLDER.value: True,
Setting.DEPRECTAED_NOTIFY_FOR_STALE_BACKUPS.value: False,
Setting.DEPRECTAED_ENABLE_BACKUP_STALE_SENSOR.value: False,
Setting.DEPRECTAED_ENABLE_BACKUP_STATE_SENSOR.value: False,
Setting.DEPRECATED_BACKUP_PASSWORD.value: "test password",
}
await ha.init()
assert not config.mustSaveUpgradeChanges()
assert interceptor.urlWasCalled(URL_MATCH_SELF_OPTIONS)
# Verify the config was upgraded
assert supervisor._options == {
Setting.MAX_BACKUPS_IN_HA.value: 1,
Setting.MAX_BACKUPS_IN_GOOGLE_DRIVE.value: 2,
Setting.DAYS_BETWEEN_BACKUPS.value: 5,
Setting.IGNORE_OTHER_BACKUPS.value: True,
Setting.IGNORE_UPGRADE_BACKUPS.value: True,
Setting.BACKUP_TIME_OF_DAY.value: "01:11",
Setting.DELETE_BEFORE_NEW_BACKUP.value: True,
Setting.BACKUP_NAME.value: "test",
Setting.SPECIFY_BACKUP_FOLDER.value: True,
Setting.NOTIFY_FOR_STALE_BACKUPS.value: False,
Setting.ENABLE_BACKUP_STALE_SENSOR.value: False,
Setting.ENABLE_BACKUP_STATE_SENSOR.value: False,
Setting.BACKUP_PASSWORD.value: "test password",
Setting.CALL_BACKUP_SNAPSHOT.value: True,
}
interceptor.clear()
await ha.init()
assert not interceptor.urlWasCalled(URL_MATCH_SELF_OPTIONS)
async def verify_upload_resumed(time,
drive: DriveSource,
config: Config,
server: SimulationServer,
interceptor: RequestInterceptor,
status,
snapshot_helper,
expected=ClientResponseError):
from_snapshot, data = await snapshot_helper.createFile()
# Configure the upload to fail after the first upload chunk
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, status, 1)
with pytest.raises(expected):
await drive.save(from_snapshot, data)
# Verify a requst was made to start the upload
assert server.wasUrlRequested(
"/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true")
assert drive.drivebackend.last_attempt_location is not None
assert drive.drivebackend.last_attempt_metadata is not None
last_location = drive.drivebackend.last_attempt_location
# Retry the upload and let is succeed
server.urls.clear()
interceptor.clear()
data.position(0)
snapshot = await drive.save(from_snapshot, data)
# We shoudl nto see the upload "initialize" url
assert not server.wasUrlRequested(
"/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true")
# We should see the last location url (which has a unique token) reused to resume the upload
assert server.wasUrlRequested(last_location)
# The saved metadata should be cleared out.
assert drive.drivebackend.last_attempt_count == 1
assert drive.drivebackend.last_attempt_location is None
assert drive.drivebackend.last_attempt_metadata is None
# Verify the uploaded bytes are identical
from_snapshot.addSource(snapshot)
download = await drive.read(from_snapshot)
data.position(0)
await compareStreams(data, download)
async def test_resume_session_reused_on_http410(time, drive: DriveSource, config: Config, server: SimulationServer, backup_helper: BackupHelper, interceptor: RequestInterceptor):
from_backup, data = await backup_helper.createFile()
# Configure the upload to fail
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 500)
with pytest.raises(GoogleInternalError):
await drive.save(from_backup, data)
# Verify a requst was made to start the upload
assert server.wasUrlRequested(URL_START_UPLOAD)
assert drive.drivebackend.last_attempt_location is not None
server.urls.clear()
interceptor.clear()
data.position(0)
interceptor.setError(drive.drivebackend.last_attempt_location, 0, 410)
await drive.save(from_backup, data)
async def test_resume_upload_attempts_exhausted(
drive: DriveSource, time, snapshot_helper,
interceptor: RequestInterceptor, google: SimulatedGoogle):
# Allow an upload to update one chunk and then fail.
from_snapshot, data = await snapshot_helper.createFile()
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, attempts=1, status=500)
with pytest.raises(GoogleInternalError):
await drive.save(from_snapshot, data)
assert google.chunks == [BASE_CHUNK_SIZE]
# Verify we have a cached location
assert drive.drivebackend.last_attempt_location is not None
assert drive.drivebackend.last_attempt_count == 1
last_location = drive.drivebackend.last_attempt_location
for x in range(1, RETRY_SESSION_ATTEMPTS):
data.position(0)
with pytest.raises(GoogleInternalError):
await drive.save(from_snapshot, data)
assert drive.drivebackend.last_attempt_count == x + 1
# We should still be using the same location url
assert drive.drivebackend.last_attempt_location == last_location
# Another attempt should use another location url
with pytest.raises(GoogleInternalError):
data.position(0)
await drive.save(from_snapshot, data)
assert drive.drivebackend.last_attempt_count == 0
assert drive.drivebackend.last_attempt_location is not None
assert drive.drivebackend.last_attempt_location != last_location
# Now let it succeed
interceptor.clear()
data.position(0)
drive_snapshot = await drive.save(from_snapshot, data)
from_snapshot.addSource(drive_snapshot)
# And verify the bytes are correct
data.position(0)
await compareStreams(data, await drive.read(from_snapshot))
async def test_resume_session_reused_on_http408(
time, drive: DriveSource, config: Config, server: SimulationServer,
snapshot_helper: SnapshotHelper, interceptor: RequestInterceptor):
from_snapshot, data = await snapshot_helper.createFile()
# Configure the upload to fail
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 408)
with pytest.raises(GoogleTimeoutError):
await drive.save(from_snapshot, data)
# Verify a requst was made to start the upload
assert server.wasUrlRequested(URL_START_UPLOAD)
location = drive.drivebackend.last_attempt_location
assert location is not None
server.urls.clear()
interceptor.clear()
data.position(0)
await drive.save(from_snapshot, data)
assert interceptor.urlWasCalled(URL(location).path)
async def test_resume_session_reused_on_http410(
time, drive: DriveSource, config: Config, server: SimulationServer,
snapshot_helper: SnapshotHelper, interceptor: RequestInterceptor):
from_snapshot, data = await snapshot_helper.createFile()
# Configure the upload to fail
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 500)
with pytest.raises(GoogleInternalError):
await drive.save(from_snapshot, data)
# Verify a requst was made to start the upload
assert server.wasUrlRequested(
"/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true")
assert drive.drivebackend.last_attempt_location is not None
server.urls.clear()
interceptor.clear()
data.position(0)
interceptor.setError(drive.drivebackend.last_attempt_location, 0, 410)
await drive.save(from_snapshot, data)
async def test_resume_session_reused_abonded_after_retries(
time, drive: DriveSource, config: Config, server, snapshot_helper,
interceptor: RequestInterceptor):
from_snapshot, data = await snapshot_helper.createFile()
# Configure the upload to fail after the first upload chunk
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1)
with pytest.raises(ClientResponseError):
await drive.save(from_snapshot, data)
# Verify a requst was made to start the upload but not cached
assert server.wasUrlRequested(
"/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true")
assert drive.drivebackend.last_attempt_count == 0
assert drive.drivebackend.last_attempt_location is not None
assert drive.drivebackend.last_attempt_metadata is not None
last_location = drive.drivebackend.last_attempt_location
for x in range(1, RETRY_SESSION_ATTEMPTS + 1):
server.urls.clear()
interceptor.clear()
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501)
data.position(0)
with pytest.raises(ClientResponseError):
await drive.save(from_snapshot, data)
assert not server.wasUrlRequested(
"/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true"
)
assert server.wasUrlRequested(last_location)
assert drive.drivebackend.last_attempt_count == x
assert drive.drivebackend.last_attempt_location is last_location
assert drive.drivebackend.last_attempt_metadata is not None
# Next attempt should give up and restart the upload
server.urls.clear()
interceptor.clear()
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1)
data.position(0)
with pytest.raises(ClientResponseError):
await drive.save(from_snapshot, data)
assert server.wasUrlRequested(
"/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true")
assert not server.wasUrlRequested(last_location)
assert drive.drivebackend.last_attempt_count == 0
# upload again, which should retry
server.urls.clear()
interceptor.clear()
data.position(0)
snapshot = await drive.save(from_snapshot, data)
assert not server.wasUrlRequested(
"/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true")
# Verify the uploaded bytes are identical
from_snapshot.addSource(snapshot)
download = await drive.read(from_snapshot)
data.position(0)
await compareStreams(data, download)
async def test_resume_session_reused_abonded_after_retries(time, drive: DriveSource, config: Config, server: SimulationServer, backup_helper, interceptor: RequestInterceptor):
from_backup, data = await backup_helper.createFile()
# Configure the upload to fail after the first upload chunk
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1)
with pytest.raises(ClientResponseError):
await drive.save(from_backup, data)
# Verify a requst was made to start the upload but not cached
assert server.wasUrlRequested(URL_START_UPLOAD)
assert drive.drivebackend.last_attempt_count == 1
assert drive.drivebackend.last_attempt_location is not None
assert drive.drivebackend.last_attempt_metadata is not None
last_location = drive.drivebackend.last_attempt_location
for x in range(1, RETRY_SESSION_ATTEMPTS):
server.urls.clear()
interceptor.clear()
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501)
data.position(0)
with pytest.raises(ClientResponseError):
await drive.save(from_backup, data)
assert not server.wasUrlRequested(URL_START_UPLOAD)
assert server.wasUrlRequested(last_location)
assert drive.drivebackend.last_attempt_count == x + 1
assert drive.drivebackend.last_attempt_location is last_location
assert drive.drivebackend.last_attempt_metadata is not None
# Next attempt should give up and restart the upload
server.urls.clear()
interceptor.clear()
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1)
data.position(0)
with pytest.raises(ClientResponseError):
await drive.save(from_backup, data)
assert server.wasUrlRequested(URL_START_UPLOAD)
assert not server.wasUrlRequested(last_location)
assert drive.drivebackend.last_attempt_count == 1
# upload again, which should retry
server.urls.clear()
interceptor.clear()
data.position(0)
backup = await drive.save(from_backup, data)
assert not server.wasUrlRequested(URL_START_UPLOAD)
# Verify the uploaded bytes are identical
from_backup.addSource(backup)
download = await drive.read(from_backup)
data.position(0)
await compareStreams(data, download)
