async def test_cred_refresh_upgrade_default_client(drive: DriveSource,
server: SimulationServer,
time: FakeTime,
config: Config):
return
# TODO: Enable this when we start removing the default client_secret
config.override(Setting.DEFAULT_DRIVE_CLIENT_ID,
server.getSetting("drive_client_id"))
creds = server.getCurrentCreds()
creds_with_secret = server.getCurrentCreds()
creds_with_secret._secret = server.getSetting("drive_client_secret")
with open(config.get(Setting.CREDENTIALS_FILE_PATH), "w") as f:
json.dump(creds_with_secret.serialize(), f)
# reload the creds
drive.drivebackend.tryLoadCredentials()
# Verify the "client secret" was removed
with open(config.get(Setting.CREDENTIALS_FILE_PATH)) as f:
saved_creds = json.load(f)
assert saved_creds == creds.serialize()
await drive.get()
old_creds = drive.drivebackend.cred_bearer
await drive.get()
assert old_creds == drive.drivebackend.cred_bearer
time.advanceDay()
await drive.get()
assert old_creds != drive.drivebackend.cred_bearer
async def test_refresh_problem_with_google(drive: DriveSource, server: SimulationServer, config: Config, time):
time.advanceDay()
server.setError(".*/oauth2/v4/token.*", status=510)
drive.drivebackend.creds._secret = None
with pytest.raises(CredRefreshGoogleError) as error:
await drive.get()
assert error.value.data() == {"from_google": "Google returned HTTP 510"}
async def test_resume_session_abandoned_after_a_long_time(
time: FakeTime, drive: DriveSource, config: Config,
server: SimulationServer, snapshot_helper,
interceptor: RequestInterceptor):
from_snapshot, data = await snapshot_helper.createFile()
# Configure the upload to fail after the first upload chunk
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1)
with pytest.raises(ClientResponseError):
await drive.save(from_snapshot, data)
# Verify it reuses the session a few times
assert server.wasUrlRequested(URL_START_UPLOAD)
assert drive.drivebackend.last_attempt_count == 1
assert drive.drivebackend.last_attempt_location is not None
assert drive.drivebackend.last_attempt_metadata is not None
data.position(0)
with pytest.raises(ClientResponseError):
await drive.save(from_snapshot, data)
assert drive.drivebackend.last_attempt_count == 2
assert drive.drivebackend.last_attempt_location is not None
assert drive.drivebackend.last_attempt_metadata is not None
last_location = drive.drivebackend.last_attempt_location
# Fast forward a lot, then verify the session is restarted
server.urls.clear()
interceptor.clear()
time.advance(duration=UPLOAD_SESSION_EXPIRATION_DURATION)
data.position(0)
await drive.save(from_snapshot, data)
assert interceptor.urlWasCalled(URL_START_UPLOAD)
assert not interceptor.urlWasCalled(last_location)
async def verify_upload_resumed(time,
drive: DriveSource,
config: Config,
server: SimulationServer,
interceptor: RequestInterceptor,
status,
snapshot_helper,
expected=ClientResponseError):
from_snapshot, data = await snapshot_helper.createFile()
# Configure the upload to fail after the first upload chunk
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, status, 1)
with pytest.raises(expected):
await drive.save(from_snapshot, data)
# Verify a requst was made to start the upload
assert server.wasUrlRequested(
"/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true")
assert drive.drivebackend.last_attempt_location is not None
assert drive.drivebackend.last_attempt_metadata is not None
last_location = drive.drivebackend.last_attempt_location
# Retry the upload and let is succeed
server.urls.clear()
interceptor.clear()
data.position(0)
snapshot = await drive.save(from_snapshot, data)
# We shoudl nto see the upload "initialize" url
assert not server.wasUrlRequested(
"/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true")
# We should see the last location url (which has a unique token) reused to resume the upload
assert server.wasUrlRequested(last_location)
# The saved metadata should be cleared out.
assert drive.drivebackend.last_attempt_count == 1
assert drive.drivebackend.last_attempt_location is None
assert drive.drivebackend.last_attempt_metadata is None
# Verify the uploaded bytes are identical
from_snapshot.addSource(snapshot)
download = await drive.read(from_snapshot)
data.position(0)
await compareStreams(data, download)
async def test_change_specify_folder_setting_with_manual_creds(
reader: ReaderHelper, server: SimulationServer, session,
coord: Coordinator, folder_finder: FolderFinder, drive: DriveSource,
config):
server.resetDriveAuth()
# Generate manual credentials
req_path = "manualauth?client_id={}&client_secret={}".format(
server.custom_drive_client_id, server.custom_drive_client_secret)
data = await reader.getjson(req_path)
assert "auth_url" in data
async with session.get(data["auth_url"], allow_redirects=False) as resp:
code = URL(resp.headers["location"]).query["code"]
drive.saveCreds(None)
assert not drive.enabled()
req_path = "manualauth?code={}".format(code)
await reader.getjson(req_path)
assert drive.isCustomCreds()
await coord.sync()
assert folder_finder.getCachedFolder() is not None
# Specify the snapshot folder, which should cache the new one
update = {
"config": {
Setting.SPECIFY_SNAPSHOT_FOLDER.value: True
},
"snapshot_folder": "12345"
}
assert await reader.postjson("saveconfig", json=update) == {
'message': 'Settings saved'
}
assert folder_finder.getCachedFolder() == "12345"
# Un change the folder, which should keep the existing folder
update = {
"config": {
Setting.SPECIFY_SNAPSHOT_FOLDER.value: False
},
"snapshot_folder": ""
}
assert await reader.postjson("saveconfig", json=update) == {
'message': 'Settings saved'
}
assert folder_finder.getCachedFolder() == "12345"
async def test_cred_refresh_no_secret(drive: DriveSource, server: SimulationServer, time: FakeTime, config: Config):
drive.saveCreds(server.getCurrentCreds())
await drive.get()
old_creds = drive.drivebackend.creds
await drive.get()
assert old_creds.access_token == drive.drivebackend.creds.access_token
time.advanceDay()
await drive.get()
assert old_creds.access_token != drive.drivebackend.creds.access_token
with open(config.get(Setting.CREDENTIALS_FILE_PATH)) as f:
assert "client_secret" not in json.load(f)
async def test_publish_retries(updater: HaUpdater, server: SimulationServer,
time: FakeTime, snapshot, drive):
await updater.update()
assert server.getEntity("sensor.snapshot_backup") is not None
# Shoudlnt update after 59 minutes
server.clearEntities()
time.advance(minutes=59)
await updater.update()
assert server.getEntity("sensor.snapshot_backup") is None
# after that it should
server.clearEntities()
time.advance(minutes=2)
await updater.update()
assert server.getEntity("sensor.snapshot_backup") is not None
server.clearEntities()
await drive.delete(snapshot)
await updater.update()
assert server.getEntity("sensor.snapshot_backup") is not None
async def test_resume_session_reused_on_http410(
time, drive: DriveSource, config: Config, server: SimulationServer,
snapshot_helper: SnapshotHelper):
from_snapshot, data = await snapshot_helper.createFile()
# Configure the upload to fail after the first upload chunk
server.setError(".*upload/drive/v3/files/progress.*", 1, 500)
with pytest.raises(GoogleInternalError):
await drive.save(from_snapshot, data)
# Verify a requst was made to start the upload
assert server.wasUrlRequested(
"/upload/drive/v3/files/?uploadType=resumable&supportsAllDrives=true")
assert drive.drivebackend.last_attempt_location is not None
server.urls.clear()
server.match_errors.clear()
data.position(0)
server.setError(drive.drivebackend.last_attempt_location, 0, 410)
await drive.save(from_snapshot, data)
async def test_download_timeout(time, drive: DriveSource, config: Config, server: SimulationServer, snapshot_helper):
config.override(Setting.DOWNLOAD_TIMEOUT_SECONDS, 1)
from_snapshot, data = await snapshot_helper.createFile()
snapshot = await drive.save(from_snapshot, data)
# Verify the uploaded bytes are identical
from_snapshot.addSource(snapshot)
server.drive_sleep = 100
download = await drive.read(from_snapshot)
data.position(0)
with pytest.raises(GoogleTimeoutError):
await compareStreams(data, download)
async def test_cred_refresh_with_secret(drive: DriveSource, server: SimulationServer, time: FakeTime, config: Config):
server.resetDriveAuth()
with open(config.get(Setting.CREDENTIALS_FILE_PATH), "w") as f:
creds = server.getCurrentCreds()
creds._secret = config.get(Setting.DEFAULT_DRIVE_CLIENT_SECRET)
json.dump(creds.serialize(), f)
drive.drivebackend.tryLoadCredentials()
await drive.get()
old_creds = drive.drivebackend.creds
# valid creds should be reused
await drive.get()
assert old_creds.access_token == drive.drivebackend.creds.access_token
# then refreshed when they expire
time.advanceDay()
await drive.get()
assert old_creds.access_token != drive.drivebackend.creds.access_token
# verify the client_secret is kept
with open(config.get(Setting.CREDENTIALS_FILE_PATH)) as f:
assert "client_secret" in json.load(f)
async def test_working_through_upload(drive: DriveSource, server: SimulationServer, snapshot_helper: SnapshotHelper):
assert not drive.isWorking()
# Let a single chunk upload, then wait
server._upload_chunk_wait.clear()
server._upload_chunk_trigger.clear()
server.waitOnChunk = 2
from_snapshot, data = await snapshot_helper.createFile(size=1024 * 1024 * 10)
save_task = asyncio.create_task(drive.save(from_snapshot, data))
await server._upload_chunk_trigger.wait()
assert drive.isWorking()
# let it complete
server._upload_chunk_wait.set()
await save_task
assert not drive.isWorking()
async def test_resume_session_reused_abonded_after_retries(
time, drive: DriveSource, config: Config, server: SimulationServer,
snapshot_helper, interceptor: RequestInterceptor):
from_snapshot, data = await snapshot_helper.createFile()
# Configure the upload to fail after the first upload chunk
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1)
with pytest.raises(ClientResponseError):
await drive.save(from_snapshot, data)
# Verify a requst was made to start the upload but not cached
assert server.wasUrlRequested(URL_START_UPLOAD)
assert drive.drivebackend.last_attempt_count == 1
assert drive.drivebackend.last_attempt_location is not None
assert drive.drivebackend.last_attempt_metadata is not None
last_location = drive.drivebackend.last_attempt_location
for x in range(1, RETRY_SESSION_ATTEMPTS):
server.urls.clear()
interceptor.clear()
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501)
data.position(0)
with pytest.raises(ClientResponseError):
await drive.save(from_snapshot, data)
assert not server.wasUrlRequested(URL_START_UPLOAD)
assert server.wasUrlRequested(last_location)
assert drive.drivebackend.last_attempt_count == x + 1
assert drive.drivebackend.last_attempt_location is last_location
assert drive.drivebackend.last_attempt_metadata is not None
# Next attempt should give up and restart the upload
server.urls.clear()
interceptor.clear()
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 501, 1)
data.position(0)
with pytest.raises(ClientResponseError):
await drive.save(from_snapshot, data)
assert server.wasUrlRequested(URL_START_UPLOAD)
assert not server.wasUrlRequested(last_location)
assert drive.drivebackend.last_attempt_count == 1
# upload again, which should retry
server.urls.clear()
interceptor.clear()
data.position(0)
snapshot = await drive.save(from_snapshot, data)
assert not server.wasUrlRequested(URL_START_UPLOAD)
# Verify the uploaded bytes are identical
from_snapshot.addSource(snapshot)
download = await drive.read(from_snapshot)
data.position(0)
await compareStreams(data, download)
async def test_resume_session_reused_on_http410(time, drive: DriveSource, config: Config, server: SimulationServer, backup_helper: BackupHelper, interceptor: RequestInterceptor):
from_backup, data = await backup_helper.createFile()
# Configure the upload to fail
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 500)
with pytest.raises(GoogleInternalError):
await drive.save(from_backup, data)
# Verify a requst was made to start the upload
assert server.wasUrlRequested(URL_START_UPLOAD)
assert drive.drivebackend.last_attempt_location is not None
server.urls.clear()
interceptor.clear()
data.position(0)
interceptor.setError(drive.drivebackend.last_attempt_location, 0, 410)
await drive.save(from_backup, data)
async def test_resume_session_reused_on_http408(
time, drive: DriveSource, config: Config, server: SimulationServer,
snapshot_helper: SnapshotHelper, interceptor: RequestInterceptor):
from_snapshot, data = await snapshot_helper.createFile()
# Configure the upload to fail
interceptor.setError(URL_MATCH_UPLOAD_PROGRESS, 408)
with pytest.raises(GoogleTimeoutError):
await drive.save(from_snapshot, data)
# Verify a requst was made to start the upload
assert server.wasUrlRequested(URL_START_UPLOAD)
location = drive.drivebackend.last_attempt_location
assert location is not None
server.urls.clear()
interceptor.clear()
data.position(0)
await drive.save(from_snapshot, data)
assert interceptor.urlWasCalled(URL(location).path)
async def test_out_of_space(snapshot_helper, drive: DriveSource, server: SimulationServer):
server.simulate_out_of_drive_space = True
from_snapshot, data = await snapshot_helper.createFile()
with pytest.raises(DriveQuotaExceeded):
await drive.save(from_snapshot, data)
