def test_status_no_db_all(self):
self.args.environments = ['course', 'master', 'system']
self.args.choose_course = None
self.args.config = SimpleNamespace()
self.args.config.database = dict()
self.args.config.submitty = dict()
self.args.config.submitty['submitty_data_dir'] = Path(self.dir)
Path(self.dir, 'courses', 'f19', 'csci1100').mkdir(parents=True)
with patch.object(migrator.db, 'Database') as mock_class:
mock_class.side_effect = OperationalError('test', None, None)
main.status(self.args)
expected = """Could not get database for migrations for master
Could not get database for migrations for system
Could not get the status for the migrations for f19.csci1100
"""
self.assertEqual(expected, sys.stdout.getvalue())
def test_save_request_to_db_raises_error_for_create_stored_request(
json_mock, close_mock):
timestamp = datetime.now(timezone.utc)
unix_timestamp = str(timestamp.timestamp())
filename = unix_timestamp + ".json"
json_response = {"foo": "bar", "baz": "gaz"}
with patch(
"fh_webhook.model_services.CreateStoredRequest.run",
side_effect=OperationalError("SELECT 1", None, None),
) as create_mock:
s = services.SaveRequestToDB(json_response, timestamp, filename).run()
assert create_mock.call_count == 1
assert json_mock.call_count == 0
assert close_mock.call_count == 0
assert s is None
async def update_spawns(self, initial=False):
while True:
try:
await run_threaded(spawns.update)
LOOP.create_task(run_threaded(spawns.pickle))
except OperationalError as e:
self.log.exception('Operational error while trying to update spawns.')
if initial:
raise OperationalError('Could not update spawns, ensure your DB is set up.') from e
await sleep(15, loop=LOOP)
except CancelledError:
raise
except Exception as e:
self.log.exception('A wild {} appeared while updating spawns!', e.__class__.__name__)
await sleep(15, loop=LOOP)
else:
break
def test_save_request_to_db_raises_error_for_process_json(
create_mock, close_mock):
timestamp = datetime.now(timezone.utc)
unix_timestamp = str(timestamp.timestamp())
filename = unix_timestamp + ".json"
json_response = {"foo": "bar", "baz": "gaz"}
with patch(
"fh_webhook.services.ProcessJSONResponse.run",
side_effect=OperationalError("SELECT 1", None, None),
) as json_mock:
s = services.SaveRequestToDB(json_response, timestamp, filename).run()
assert create_mock.call_count == 1
assert json_mock.call_count == 1
assert close_mock.call_count == 0 # as the exception avoided this call
assert s is None
def test_retry_db_transaction_with_passing_retries(self):
"""Test that retries can be passed to decorator"""
mock_obj = mock.MagicMock()
mock_session = mock.MagicMock()
op_error = OperationalError(statement=mock.ANY,
params=mock.ANY,
orig=mock.ANY)
@retry_db_transaction(retries=2)
def test_function(session):
session.execute("select 1")
mock_obj(2)
raise op_error
with self.assertRaises(OperationalError):
test_function(session=mock_session)
assert mock_obj.call_count == 2
def test_migration_no_db_all(self):
args = Namespace()
args.environments = ['course', 'master', 'system']
args.choose_course = None
args.config = SimpleNamespace()
args.config.database = dict()
args.config.submitty = dict()
args.config.submitty['submitty_data_dir'] = Path(self.dir)
Path(self.dir, 'courses', 'f19', 'csci1100').mkdir(parents=True)
with patch.object(migrator.db, 'Database') as mock_class:
mock_class.side_effect = OperationalError('test', None, None)
main.handle_migration(args)
expected = """Database does not exist for master
Database does not exist for system
Submitty Database Migration Warning: Database does not exist for semester=f19 course=csci1100
"""
self.assertEqual(expected, sys.stdout.getvalue())
def test_000_check_status_error(self, mock_response, mock_session,
mock_senderror, mock_sendresult):
"""
Test that 'check_status' returns an error status code
"""
op_error = OperationalError(statement="Error", params={}, orig="Error")
mock_session.query.side_effect = op_error
mock_senderror.side_effect = lambda response, exx: exx
mock_sendresult.side_effect = lambda response, obj, *args: obj
with self.assertRaises(HTTPInternalServerError) as err:
self.maint.check_status()
self.assertTrue(err.exception.status_code == 500)
return
def read_sql(sql,
con,
index_col=None,
coerce_float=True,
params=None,
parse_dates=None,
columns=None,
chunksize=None,
*args,
**kwargs) -> pd.DataFrame:
if kwargs.get("raise_sqlalchemy_error"):
raise SQLAlchemyError("error")
elif kwargs.get("raise_operational_error"):
raise OperationalError(statement="error",
params={"error": "error"},
orig="error")
else:
return PANDAS_DATAFRAME
def test_migration_no_db_all(self):
args = Namespace()
args.environments = ['course', 'master', 'system']
args.choose_course = None
args.config = SimpleNamespace()
args.config.database = dict()
args.config.submitty = dict()
args.config.submitty['submitty_data_dir'] = Path(self.dir)
Path(self.dir, 'courses', 'f19', 'csci1100').mkdir(parents=True)
with self.assertRaises(SystemExit) as context, \
patch.object(migrator.db, 'Database') as mock_class:
mock_class.side_effect = OperationalError('test', None, None)
main.handle_migration(args)
self.assertEqual(
"Submitty Database Migration Error: Database does not exist for master",
str(context.exception))
def to_sql(name,
con,
schema=None,
if_exists="fail",
index=True,
index_label=None,
chunksize=None,
dtype=None,
method=None,
*args,
**kwargs):
if kwargs.get("raise_sqlalchemy_error"):
raise SQLAlchemyError("error")
elif kwargs.get("raise_operational_error"):
raise OperationalError(statement="error",
params={"error": "error"},
orig="error")
else:
pass
def test_api_post_pubsub_deadlock_is_409(dbsession, pubsub_client):
"""A deadlock is turned into a 409 Conflict"""
data = stripe_customer_data()
err = OperationalError("INSERT INTO...", {"stripe_id": data["id"]},
"Deadlock")
with capture_logs() as caplog, mock.patch.object(dbsession,
"commit",
side_effect=err):
resp = pubsub_client.post("/stripe_from_pubsub",
json=pubsub_wrap(data))
assert resp.status_code == 409
assert resp.json() == {"detail": "Deadlock or other issue, try again"}
assert len(caplog) == 2
assert caplog[0] == {
"exc_info": True,
"event": "OperationalError converted to 409",
"log_level": "error",
}
assert caplog[1]["status_code"] == 409
def test_sync_to_db_is_retried(self, mock_bulk_write_to_db,
mock_s10n_write_dag, mock_collect_dags):
"""Test that dagbag.sync_to_db is retried on OperationalError"""
dagbag = DagBag("/dev/null")
mock_dag = mock.MagicMock(spec=models.DAG)
mock_dag.is_subdag = False
dagbag.dags['mock_dag'] = mock_dag
op_error = OperationalError(statement=mock.ANY,
params=mock.ANY,
orig=mock.ANY)
# Mock error for the first 2 tries and a successful third try
side_effect = [op_error, op_error, mock.ANY]
mock_bulk_write_to_db.side_effect = side_effect
mock_session = mock.MagicMock()
dagbag.sync_to_db(session=mock_session)
# Test that 3 attempts were made to run 'DAG.bulk_write_to_db' successfully
mock_bulk_write_to_db.assert_has_calls([
mock.call(mock.ANY, session=mock.ANY),
mock.call(mock.ANY, session=mock.ANY),
mock.call(mock.ANY, session=mock.ANY),
])
# Assert that rollback is called twice (i.e. whenever OperationalError occurs)
mock_session.rollback.assert_has_calls([mock.call(), mock.call()])
# Check that 'SerializedDagModel.write_dag' is also called
# Only called once since the other two times the 'DAG.bulk_write_to_db' error'd
# and the session was roll-backed before even reaching 'SerializedDagModel.write_dag'
mock_s10n_write_dag.assert_has_calls([
mock.call(mock_dag,
min_update_interval=mock.ANY,
session=mock_session),
])
def test_retry_db_transaction_with_default_retries(self):
"""Test that by default 3 retries will be carried out"""
mock_obj = mock.MagicMock()
mock_session = mock.MagicMock()
mock_rollback = mock.MagicMock()
mock_session.rollback = mock_rollback
op_error = OperationalError(statement=mock.ANY,
params=mock.ANY,
orig=mock.ANY)
@retry_db_transaction
def test_function(session):
session.execute("select 1")
mock_obj(2)
raise op_error
with self.assertRaises(OperationalError), self.assertLogs(
self.__module__, 'DEBUG') as logs_output:
test_function(session=mock_session)
assert (
"DEBUG:tests.utils.test_retries:Running "
"TestRetries.test_retry_db_transaction_with_default_retries.<locals>.test_function "
"with retries. Try 1 of 3" in logs_output.output)
assert (
"DEBUG:tests.utils.test_retries:Running "
"TestRetries.test_retry_db_transaction_with_default_retries.<locals>.test_function "
"with retries. Try 2 of 3" in logs_output.output)
assert (
"DEBUG:tests.utils.test_retries:Running "
"TestRetries.test_retry_db_transaction_with_default_retries.<locals>.test_function "
"with retries. Try 3 of 3" in logs_output.output)
assert mock_session.execute.call_count == 3
assert mock_rollback.call_count == 3
mock_rollback.assert_has_calls([mock.call(), mock.call(), mock.call()])
def launch(self, bootstrap, pickle):
initial = True
while not self.killed:
if not initial:
pickle = False
bootstrap = False
while True:
try:
self.spawns.update(loadpickle=pickle)
except OperationalError as e:
self.logger.exception('Operational error while trying to update spawns.')
if initial:
_thread.interrupt_main()
raise OperationalError('Could not update spawns, ensure your DB is setup.') from e
time.sleep(20)
except Exception:
self.logger.exception('A wild exception occurred while updating spawns.')
time.sleep(20)
else:
break
if not self.spawns or bootstrap:
bootstrap = True
pickle = False
if bootstrap:
self.bootstrap()
while len(self.spawns) < 10 and not self.killed:
try:
mystery_point = list(self.mysteries.popleft())
self.coroutine_semaphore.acquire()
asyncio.run_coroutine_threadsafe(
self.try_point(mystery_point), loop=self.loop
)
except IndexError:
if self.spawns.mysteries:
self.mysteries = self.spawns.get_mysteries()
else:
config.MORE_POINTS = True
break
current_hour = get_current_hour()
if self.spawns.after_last():
current_hour += 3600
initial = False
if initial:
start_point = self.get_start_point()
if not start_point:
initial = False
else:
dump_pickle('accounts', self.accounts)
for spawn_id, spawn in self.spawns.items():
try:
if initial:
if spawn_id == start_point:
initial = False
else:
continue
if self.captcha_queue.qsize() > config.MAX_CAPTCHAS:
self.paused = True
try:
self.idle_seconds += self.captcha_queue.full_wait(
maxsize=config.MAX_CAPTCHAS)
except (EOFError, BrokenPipeError, FileNotFoundError):
pass
self.paused = False
point = list(spawn[0])
spawn_time = spawn[1] + current_hour
# negative = hasn't happened yet
# positive = already happened
time_diff = time.time() - spawn_time
while time_diff < 0 and not self.killed:
try:
mystery_point = list(self.mysteries.popleft())
self.coroutine_semaphore.acquire()
asyncio.run_coroutine_threadsafe(
self.try_point(mystery_point), loop=self.loop
)
except IndexError:
if self.spawns.mysteries:
self.mysteries = self.spawns.get_mysteries()
else:
config.MORE_POINTS = True
break
time_diff = time.time() - spawn_time
if time_diff > 5 and spawn_id in SIGHTING_CACHE.spawns:
self.redundant += 1
continue
elif time_diff > config.SKIP_SPAWN:
self.skipped += 1
continue
if self.killed:
return
self.coroutine_semaphore.acquire()
asyncio.run_coroutine_threadsafe(
self.try_point(point, spawn_time), loop=self.loop
)
except Exception:
self.logger.exception('Error occured in launcher loop.')
from .test_company import company
from unittest.mock import patch
from app.views.product import (logical_delete, logical_restore, find_by_id,
create, update, find)
from random import randrange
from sqlalchemy.exc import (NotSupportedError, OperationalError,
ProgrammingError, IntegrityError, InternalError,
DataError)
from .test_user import user
import unittest
product = Product('id', company_id='company_id', value=10.00)
integrity_error = IntegrityError('Mock', 'mock', Exception('mock', 'mock'))
aleatory_errors = [
DataError('Mock', 'mock', Exception('mock', 'mock')),
OperationalError('Mock', 'mock', Exception('mock', 'mock')),
InternalError('Mock', 'mock', Exception('mock', 'mock')),
ProgrammingError('Mock', 'mock', Exception('mock', 'mock')),
NotSupportedError('Mock', 'mock', Exception('mock', 'mock'))
]
product_validate = Schema({
'id': Use(str),
'company_id': Use(str),
'value': Use(float),
'created': Or(str, None),
'updated': Or(str, None),
'removed': Or(str, None)
})
companies_products_validate = Schema([{
def raise_operational_error(_):
raise OperationalError(statement='Fail', params=None, orig=None)
def _op_exc(connection_invalidated=False):
return OperationalError(
None, None, None, connection_invalidated=connection_invalidated)
assert json_data["id"] == 1
assert json_data["title"] == "test"
assert json_data["meta"] == '["meta", "testing"]'
assert json.loads(json_data["meta"]) == ["meta", "testing"]
assert json_data["description"] == "some testing"
assert json_data["image"] == "image.png"
assert json_data["git_repo"] == "github.com"
assert json_data["demo_link"] == "heroku.com"
assert res.status_code == 200
@patch(
"src.apis.projects.Project",
MagicMock(
**{"query.all.side_effect": OperationalError("err", "err", "err")}),
)
def test_api_project_get_db_error(app):
"""Test api GET /api/project endpoint db error"""
with app.test_client() as client:
res = client.get("/api/project")
json_data = res.get_json()
assert json_data["success"] is False
assert json_data["error"] == 502
assert "invalid response from an upstream server" in json_data["message"]
@patch(
"src.apis.projects.Project",
MagicMock(**{"query.all.side_effect": Exception}),
from sqlalchemy.exc import OperationalError
from unittest.mock import patch
from test.utils import db
from test.db_mock import create_new_file
@patch("app.file.db.session.commit",
side_effect=[OperationalError(None, None, None), None])
def test_retry(db_commit_mock):
# GIVEN
file = create_new_file(db)
stats = {
"percent": 34.7,
"size": "102.16MiB",
"speed": "144.30KiB/s",
"time_remaining": "07:53",
}
# WHEN
file.update_db(stats)
# THEN
assert 2 == db_commit_mock.call_count
def raise_error():
raise OperationalError(None, None, None, connection_invalidated=False)
def _throw_if_state_in_session(*args, **kwargs):
for obj in opp.data[DATA_INSTANCE].event_session:
if isinstance(obj, States):
raise OperationalError("insert the state", "fake params",
"forced to fail")
def create_all():
raise OperationalError("a", "a", "a")
def test_retry_if_data_catalog_exception(self, engine, conn):
dialect = engine.dialect
exc = OperationalError(
'', None,
'Database does_not_exist not found. Please check your query.')
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, 'does_not_exist',
'does_not_exist'))
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, 'does_not_exist',
'this_does_not_exist'))
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc,
'this_does_not_exist',
'does_not_exist'))
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc,
'this_does_not_exist',
'this_does_not_exist'))
exc = OperationalError(
'', None,
'Namespace does_not_exist not found. Please check your query.')
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, 'does_not_exist',
'does_not_exist'))
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, 'does_not_exist',
'this_does_not_exist'))
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc,
'this_does_not_exist',
'does_not_exist'))
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc,
'this_does_not_exist',
'this_does_not_exist'))
exc = OperationalError(
'', None,
'Table does_not_exist not found. Please check your query.')
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, 'does_not_exist',
'does_not_exist'))
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc, 'does_not_exist',
'this_does_not_exist'))
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc,
'this_does_not_exist',
'does_not_exist'))
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc,
'this_does_not_exist',
'this_does_not_exist'))
exc = OperationalError('', None, 'foobar.')
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc, 'foobar', 'foobar'))
exc = ProgrammingError(
'', None,
'Database does_not_exist not found. Please check your query.')
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, 'does_not_exist',
'does_not_exist'))
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, 'does_not_exist',
'this_does_not_exist'))
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc,
'this_does_not_exist',
'does_not_exist'))
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc,
'this_does_not_exist',
'this_does_not_exist'))
def raise_exc(account):
if attempts[0] == 0:
attempts[0] += 1
raise OperationalError('', '', Struct(args=[2006]))
return original(account)
'lastname': 'User',
'experience': [{
'company': 'Firma',
'project': 'Project',
'duration': 5
}]
}, None, AttributeError, ({
'error': 'bad input data'
}, 400)),
({
'username': '******',
'firstname': 'no skills',
'lastname': 'User',
'skills': [],
'experience': []
}, OperationalError('', '', ''), None, ({
'error': 'db error'
}, 500)), ({}, None, None, ({
'error': 'bad input data'
}, 400))])
@patch('app.auth.extract_user')
@patch('app.main.get_session')
@patch('app.main.replace_skills_with_json')
@patch('app.main.replace_experience_with_json')
def test_api_post_cv(replace_experience_with_json_mock,
replace_skills_with_json_mock, get_session_mock,
extract_user_mock, api_cv_post, client, test_input,
replace_skills_with_json_error,
replace_experience_with_json_error, expected):
if replace_skills_with_json_error is not None:
replace_skills_with_json_mock.side_effect = replace_skills_with_json_error
def errors(x):
raise OperationalError(None, None, None, None)
async def test_database_corruption_while_running(opp, tmpdir, caplog):
"""Test we can recover from sqlite3 db corruption."""
def _create_tmpdir_for_test_db():
return tmpdir.mkdir("sqlite").join("test.db")
test_db_file = await opp.async_add_executor_job(_create_tmpdir_for_test_db)
dburl = f"{SQLITE_URL_PREFIX}//{test_db_file}"
assert await async_setup_component(opp, DOMAIN,
{DOMAIN: {
CONF_DB_URL: dburl
}})
await opp.async_block_till_done()
caplog.clear()
opp.states.async_set("test.lost", "on", {})
sqlite3_exception = DatabaseError("statement", {}, [])
sqlite3_exception.__cause__ = sqlite3.DatabaseError()
with patch.object(
opp.data[DATA_INSTANCE].event_session,
"close",
side_effect=OperationalError("statement", {}, []),
):
await async_wait_recording_done_without_instance(opp)
await opp.async_add_executor_job(corrupt_db_file, test_db_file)
await async_wait_recording_done_without_instance(opp)
with patch.object(
opp.data[DATA_INSTANCE].event_session,
"commit",
side_effect=[sqlite3_exception, None],
):
# This state will not be recorded because
# the database corruption will be discovered
# and we will have to rollback to recover
opp.states.async_set("test.one", "off", {})
await async_wait_recording_done_without_instance(opp)
assert "Unrecoverable sqlite3 database corruption detected" in caplog.text
assert "The system will rename the corrupt database file" in caplog.text
assert "Connected to recorder database" in caplog.text
# This state should go into the new database
opp.states.async_set("test.two", "on", {})
await async_wait_recording_done_without_instance(opp)
def _get_last_state():
with session_scope(opp=opp) as session:
db_states = list(session.query(States))
assert len(db_states) == 1
assert db_states[0].event_id > 0
return db_states[0].to_native()
state = await opp.async_add_executor_job(_get_last_state)
assert state.entity_id == "test.two"
assert state.state == "on"
opp.bus.async_fire(EVENT_OPENPEERPOWER_STOP)
await opp.async_block_till_done()
opp.stop()
def test_function(session):
session.execute("select 1;")
raise OperationalError(statement=mock.ANY,
params=mock.ANY,
orig=mock.ANY)
def test_retry_if_data_catalog_exception(self, engine, conn):
dialect = engine.dialect
exc = OperationalError(
"", None,
"Database does_not_exist not found. Please check your query.")
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, "does_not_exist",
"does_not_exist"))
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, "does_not_exist",
"this_does_not_exist"))
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc,
"this_does_not_exist",
"does_not_exist"))
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc,
"this_does_not_exist",
"this_does_not_exist"))
exc = OperationalError(
"", None,
"Namespace does_not_exist not found. Please check your query.")
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, "does_not_exist",
"does_not_exist"))
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, "does_not_exist",
"this_does_not_exist"))
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc,
"this_does_not_exist",
"does_not_exist"))
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc,
"this_does_not_exist",
"this_does_not_exist"))
exc = OperationalError(
"", None,
"Table does_not_exist not found. Please check your query.")
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, "does_not_exist",
"does_not_exist"))
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc, "does_not_exist",
"this_does_not_exist"))
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc,
"this_does_not_exist",
"does_not_exist"))
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc,
"this_does_not_exist",
"this_does_not_exist"))
exc = OperationalError("", None, "foobar.")
self.assertTrue(
dialect._retry_if_data_catalog_exception(exc, "foobar", "foobar"))
exc = ProgrammingError(
"", None,
"Database does_not_exist not found. Please check your query.")
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, "does_not_exist",
"does_not_exist"))
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc, "does_not_exist",
"this_does_not_exist"))
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc,
"this_does_not_exist",
"does_not_exist"))
self.assertFalse(
dialect._retry_if_data_catalog_exception(exc,
"this_does_not_exist",
"this_does_not_exist"))
def raiser():
raise OperationalError("foo", {}, psycopg2.OperationalError())
