def test_status_no_db_all(self): self.args.environments = ['course', 'master', 'system'] self.args.choose_course = None self.args.config = SimpleNamespace() self.args.config.database = dict() self.args.config.submitty = dict() self.args.config.submitty['submitty_data_dir'] = Path(self.dir) Path(self.dir, 'courses', 'f19', 'csci1100').mkdir(parents=True) with patch.object(migrator.db, 'Database') as mock_class: mock_class.side_effect = OperationalError('test', None, None) main.status(self.args) expected = """Could not get database for migrations for master Could not get database for migrations for system Could not get the status for the migrations for f19.csci1100 """ self.assertEqual(expected, sys.stdout.getvalue())
def test_save_request_to_db_raises_error_for_create_stored_request( json_mock, close_mock): timestamp = datetime.now(timezone.utc) unix_timestamp = str(timestamp.timestamp()) filename = unix_timestamp + ".json" json_response = {"foo": "bar", "baz": "gaz"} with patch( "fh_webhook.model_services.CreateStoredRequest.run", side_effect=OperationalError("SELECT 1", None, None), ) as create_mock: s = services.SaveRequestToDB(json_response, timestamp, filename).run() assert create_mock.call_count == 1 assert json_mock.call_count == 0 assert close_mock.call_count == 0 assert s is None
async def update_spawns(self, initial=False): while True: try: await run_threaded(spawns.update) LOOP.create_task(run_threaded(spawns.pickle)) except OperationalError as e: self.log.exception('Operational error while trying to update spawns.') if initial: raise OperationalError('Could not update spawns, ensure your DB is set up.') from e await sleep(15, loop=LOOP) except CancelledError: raise except Exception as e: self.log.exception('A wild {} appeared while updating spawns!', e.__class__.__name__) await sleep(15, loop=LOOP) else: break
def test_save_request_to_db_raises_error_for_process_json( create_mock, close_mock): timestamp = datetime.now(timezone.utc) unix_timestamp = str(timestamp.timestamp()) filename = unix_timestamp + ".json" json_response = {"foo": "bar", "baz": "gaz"} with patch( "fh_webhook.services.ProcessJSONResponse.run", side_effect=OperationalError("SELECT 1", None, None), ) as json_mock: s = services.SaveRequestToDB(json_response, timestamp, filename).run() assert create_mock.call_count == 1 assert json_mock.call_count == 1 assert close_mock.call_count == 0 # as the exception avoided this call assert s is None
def test_retry_db_transaction_with_passing_retries(self): """Test that retries can be passed to decorator""" mock_obj = mock.MagicMock() mock_session = mock.MagicMock() op_error = OperationalError(statement=mock.ANY, params=mock.ANY, orig=mock.ANY) @retry_db_transaction(retries=2) def test_function(session): session.execute("select 1") mock_obj(2) raise op_error with self.assertRaises(OperationalError): test_function(session=mock_session) assert mock_obj.call_count == 2
def test_migration_no_db_all(self): args = Namespace() args.environments = ['course', 'master', 'system'] args.choose_course = None args.config = SimpleNamespace() args.config.database = dict() args.config.submitty = dict() args.config.submitty['submitty_data_dir'] = Path(self.dir) Path(self.dir, 'courses', 'f19', 'csci1100').mkdir(parents=True) with patch.object(migrator.db, 'Database') as mock_class: mock_class.side_effect = OperationalError('test', None, None) main.handle_migration(args) expected = """Database does not exist for master Database does not exist for system Submitty Database Migration Warning: Database does not exist for semester=f19 course=csci1100 """ self.assertEqual(expected, sys.stdout.getvalue())
def test_000_check_status_error(self, mock_response, mock_session, mock_senderror, mock_sendresult): """ Test that 'check_status' returns an error status code """ op_error = OperationalError(statement="Error", params={}, orig="Error") mock_session.query.side_effect = op_error mock_senderror.side_effect = lambda response, exx: exx mock_sendresult.side_effect = lambda response, obj, *args: obj with self.assertRaises(HTTPInternalServerError) as err: self.maint.check_status() self.assertTrue(err.exception.status_code == 500) return
def read_sql(sql, con, index_col=None, coerce_float=True, params=None, parse_dates=None, columns=None, chunksize=None, *args, **kwargs) -> pd.DataFrame: if kwargs.get("raise_sqlalchemy_error"): raise SQLAlchemyError("error") elif kwargs.get("raise_operational_error"): raise OperationalError(statement="error", params={"error": "error"}, orig="error") else: return PANDAS_DATAFRAME
def test_migration_no_db_all(self): args = Namespace() args.environments = ['course', 'master', 'system'] args.choose_course = None args.config = SimpleNamespace() args.config.database = dict() args.config.submitty = dict() args.config.submitty['submitty_data_dir'] = Path(self.dir) Path(self.dir, 'courses', 'f19', 'csci1100').mkdir(parents=True) with self.assertRaises(SystemExit) as context, \ patch.object(migrator.db, 'Database') as mock_class: mock_class.side_effect = OperationalError('test', None, None) main.handle_migration(args) self.assertEqual( "Submitty Database Migration Error: Database does not exist for master", str(context.exception))
def to_sql(name, con, schema=None, if_exists="fail", index=True, index_label=None, chunksize=None, dtype=None, method=None, *args, **kwargs): if kwargs.get("raise_sqlalchemy_error"): raise SQLAlchemyError("error") elif kwargs.get("raise_operational_error"): raise OperationalError(statement="error", params={"error": "error"}, orig="error") else: pass
def test_api_post_pubsub_deadlock_is_409(dbsession, pubsub_client): """A deadlock is turned into a 409 Conflict""" data = stripe_customer_data() err = OperationalError("INSERT INTO...", {"stripe_id": data["id"]}, "Deadlock") with capture_logs() as caplog, mock.patch.object(dbsession, "commit", side_effect=err): resp = pubsub_client.post("/stripe_from_pubsub", json=pubsub_wrap(data)) assert resp.status_code == 409 assert resp.json() == {"detail": "Deadlock or other issue, try again"} assert len(caplog) == 2 assert caplog[0] == { "exc_info": True, "event": "OperationalError converted to 409", "log_level": "error", } assert caplog[1]["status_code"] == 409
def test_sync_to_db_is_retried(self, mock_bulk_write_to_db, mock_s10n_write_dag, mock_collect_dags): """Test that dagbag.sync_to_db is retried on OperationalError""" dagbag = DagBag("/dev/null") mock_dag = mock.MagicMock(spec=models.DAG) mock_dag.is_subdag = False dagbag.dags['mock_dag'] = mock_dag op_error = OperationalError(statement=mock.ANY, params=mock.ANY, orig=mock.ANY) # Mock error for the first 2 tries and a successful third try side_effect = [op_error, op_error, mock.ANY] mock_bulk_write_to_db.side_effect = side_effect mock_session = mock.MagicMock() dagbag.sync_to_db(session=mock_session) # Test that 3 attempts were made to run 'DAG.bulk_write_to_db' successfully mock_bulk_write_to_db.assert_has_calls([ mock.call(mock.ANY, session=mock.ANY), mock.call(mock.ANY, session=mock.ANY), mock.call(mock.ANY, session=mock.ANY), ]) # Assert that rollback is called twice (i.e. whenever OperationalError occurs) mock_session.rollback.assert_has_calls([mock.call(), mock.call()]) # Check that 'SerializedDagModel.write_dag' is also called # Only called once since the other two times the 'DAG.bulk_write_to_db' error'd # and the session was roll-backed before even reaching 'SerializedDagModel.write_dag' mock_s10n_write_dag.assert_has_calls([ mock.call(mock_dag, min_update_interval=mock.ANY, session=mock_session), ])
def test_retry_db_transaction_with_default_retries(self): """Test that by default 3 retries will be carried out""" mock_obj = mock.MagicMock() mock_session = mock.MagicMock() mock_rollback = mock.MagicMock() mock_session.rollback = mock_rollback op_error = OperationalError(statement=mock.ANY, params=mock.ANY, orig=mock.ANY) @retry_db_transaction def test_function(session): session.execute("select 1") mock_obj(2) raise op_error with self.assertRaises(OperationalError), self.assertLogs( self.__module__, 'DEBUG') as logs_output: test_function(session=mock_session) assert ( "DEBUG:tests.utils.test_retries:Running " "TestRetries.test_retry_db_transaction_with_default_retries.<locals>.test_function " "with retries. Try 1 of 3" in logs_output.output) assert ( "DEBUG:tests.utils.test_retries:Running " "TestRetries.test_retry_db_transaction_with_default_retries.<locals>.test_function " "with retries. Try 2 of 3" in logs_output.output) assert ( "DEBUG:tests.utils.test_retries:Running " "TestRetries.test_retry_db_transaction_with_default_retries.<locals>.test_function " "with retries. Try 3 of 3" in logs_output.output) assert mock_session.execute.call_count == 3 assert mock_rollback.call_count == 3 mock_rollback.assert_has_calls([mock.call(), mock.call(), mock.call()])
def launch(self, bootstrap, pickle): initial = True while not self.killed: if not initial: pickle = False bootstrap = False while True: try: self.spawns.update(loadpickle=pickle) except OperationalError as e: self.logger.exception('Operational error while trying to update spawns.') if initial: _thread.interrupt_main() raise OperationalError('Could not update spawns, ensure your DB is setup.') from e time.sleep(20) except Exception: self.logger.exception('A wild exception occurred while updating spawns.') time.sleep(20) else: break if not self.spawns or bootstrap: bootstrap = True pickle = False if bootstrap: self.bootstrap() while len(self.spawns) < 10 and not self.killed: try: mystery_point = list(self.mysteries.popleft()) self.coroutine_semaphore.acquire() asyncio.run_coroutine_threadsafe( self.try_point(mystery_point), loop=self.loop ) except IndexError: if self.spawns.mysteries: self.mysteries = self.spawns.get_mysteries() else: config.MORE_POINTS = True break current_hour = get_current_hour() if self.spawns.after_last(): current_hour += 3600 initial = False if initial: start_point = self.get_start_point() if not start_point: initial = False else: dump_pickle('accounts', self.accounts) for spawn_id, spawn in self.spawns.items(): try: if initial: if spawn_id == start_point: initial = False else: continue if self.captcha_queue.qsize() > config.MAX_CAPTCHAS: self.paused = True try: self.idle_seconds += self.captcha_queue.full_wait( maxsize=config.MAX_CAPTCHAS) except (EOFError, BrokenPipeError, FileNotFoundError): pass self.paused = False point = list(spawn[0]) spawn_time = spawn[1] + current_hour # negative = hasn't happened yet # positive = already happened time_diff = time.time() - spawn_time while time_diff < 0 and not self.killed: try: mystery_point = list(self.mysteries.popleft()) self.coroutine_semaphore.acquire() asyncio.run_coroutine_threadsafe( self.try_point(mystery_point), loop=self.loop ) except IndexError: if self.spawns.mysteries: self.mysteries = self.spawns.get_mysteries() else: config.MORE_POINTS = True break time_diff = time.time() - spawn_time if time_diff > 5 and spawn_id in SIGHTING_CACHE.spawns: self.redundant += 1 continue elif time_diff > config.SKIP_SPAWN: self.skipped += 1 continue if self.killed: return self.coroutine_semaphore.acquire() asyncio.run_coroutine_threadsafe( self.try_point(point, spawn_time), loop=self.loop ) except Exception: self.logger.exception('Error occured in launcher loop.')
from .test_company import company from unittest.mock import patch from app.views.product import (logical_delete, logical_restore, find_by_id, create, update, find) from random import randrange from sqlalchemy.exc import (NotSupportedError, OperationalError, ProgrammingError, IntegrityError, InternalError, DataError) from .test_user import user import unittest product = Product('id', company_id='company_id', value=10.00) integrity_error = IntegrityError('Mock', 'mock', Exception('mock', 'mock')) aleatory_errors = [ DataError('Mock', 'mock', Exception('mock', 'mock')), OperationalError('Mock', 'mock', Exception('mock', 'mock')), InternalError('Mock', 'mock', Exception('mock', 'mock')), ProgrammingError('Mock', 'mock', Exception('mock', 'mock')), NotSupportedError('Mock', 'mock', Exception('mock', 'mock')) ] product_validate = Schema({ 'id': Use(str), 'company_id': Use(str), 'value': Use(float), 'created': Or(str, None), 'updated': Or(str, None), 'removed': Or(str, None) }) companies_products_validate = Schema([{
def raise_operational_error(_): raise OperationalError(statement='Fail', params=None, orig=None)
def _op_exc(connection_invalidated=False): return OperationalError( None, None, None, connection_invalidated=connection_invalidated)
assert json_data["id"] == 1 assert json_data["title"] == "test" assert json_data["meta"] == '["meta", "testing"]' assert json.loads(json_data["meta"]) == ["meta", "testing"] assert json_data["description"] == "some testing" assert json_data["image"] == "image.png" assert json_data["git_repo"] == "github.com" assert json_data["demo_link"] == "heroku.com" assert res.status_code == 200 @patch( "src.apis.projects.Project", MagicMock( **{"query.all.side_effect": OperationalError("err", "err", "err")}), ) def test_api_project_get_db_error(app): """Test api GET /api/project endpoint db error""" with app.test_client() as client: res = client.get("/api/project") json_data = res.get_json() assert json_data["success"] is False assert json_data["error"] == 502 assert "invalid response from an upstream server" in json_data["message"] @patch( "src.apis.projects.Project", MagicMock(**{"query.all.side_effect": Exception}),
from sqlalchemy.exc import OperationalError from unittest.mock import patch from test.utils import db from test.db_mock import create_new_file @patch("app.file.db.session.commit", side_effect=[OperationalError(None, None, None), None]) def test_retry(db_commit_mock): # GIVEN file = create_new_file(db) stats = { "percent": 34.7, "size": "102.16MiB", "speed": "144.30KiB/s", "time_remaining": "07:53", } # WHEN file.update_db(stats) # THEN assert 2 == db_commit_mock.call_count
def raise_error(): raise OperationalError(None, None, None, connection_invalidated=False)
def _throw_if_state_in_session(*args, **kwargs): for obj in opp.data[DATA_INSTANCE].event_session: if isinstance(obj, States): raise OperationalError("insert the state", "fake params", "forced to fail")
def create_all(): raise OperationalError("a", "a", "a")
def test_retry_if_data_catalog_exception(self, engine, conn): dialect = engine.dialect exc = OperationalError( '', None, 'Database does_not_exist not found. Please check your query.') self.assertFalse( dialect._retry_if_data_catalog_exception(exc, 'does_not_exist', 'does_not_exist')) self.assertFalse( dialect._retry_if_data_catalog_exception(exc, 'does_not_exist', 'this_does_not_exist')) self.assertTrue( dialect._retry_if_data_catalog_exception(exc, 'this_does_not_exist', 'does_not_exist')) self.assertTrue( dialect._retry_if_data_catalog_exception(exc, 'this_does_not_exist', 'this_does_not_exist')) exc = OperationalError( '', None, 'Namespace does_not_exist not found. Please check your query.') self.assertFalse( dialect._retry_if_data_catalog_exception(exc, 'does_not_exist', 'does_not_exist')) self.assertFalse( dialect._retry_if_data_catalog_exception(exc, 'does_not_exist', 'this_does_not_exist')) self.assertTrue( dialect._retry_if_data_catalog_exception(exc, 'this_does_not_exist', 'does_not_exist')) self.assertTrue( dialect._retry_if_data_catalog_exception(exc, 'this_does_not_exist', 'this_does_not_exist')) exc = OperationalError( '', None, 'Table does_not_exist not found. Please check your query.') self.assertFalse( dialect._retry_if_data_catalog_exception(exc, 'does_not_exist', 'does_not_exist')) self.assertTrue( dialect._retry_if_data_catalog_exception(exc, 'does_not_exist', 'this_does_not_exist')) self.assertFalse( dialect._retry_if_data_catalog_exception(exc, 'this_does_not_exist', 'does_not_exist')) self.assertTrue( dialect._retry_if_data_catalog_exception(exc, 'this_does_not_exist', 'this_does_not_exist')) exc = OperationalError('', None, 'foobar.') self.assertTrue( dialect._retry_if_data_catalog_exception(exc, 'foobar', 'foobar')) exc = ProgrammingError( '', None, 'Database does_not_exist not found. Please check your query.') self.assertFalse( dialect._retry_if_data_catalog_exception(exc, 'does_not_exist', 'does_not_exist')) self.assertFalse( dialect._retry_if_data_catalog_exception(exc, 'does_not_exist', 'this_does_not_exist')) self.assertFalse( dialect._retry_if_data_catalog_exception(exc, 'this_does_not_exist', 'does_not_exist')) self.assertFalse( dialect._retry_if_data_catalog_exception(exc, 'this_does_not_exist', 'this_does_not_exist'))
def raise_exc(account): if attempts[0] == 0: attempts[0] += 1 raise OperationalError('', '', Struct(args=[2006])) return original(account)
'lastname': 'User', 'experience': [{ 'company': 'Firma', 'project': 'Project', 'duration': 5 }] }, None, AttributeError, ({ 'error': 'bad input data' }, 400)), ({ 'username': '******', 'firstname': 'no skills', 'lastname': 'User', 'skills': [], 'experience': [] }, OperationalError('', '', ''), None, ({ 'error': 'db error' }, 500)), ({}, None, None, ({ 'error': 'bad input data' }, 400))]) @patch('app.auth.extract_user') @patch('app.main.get_session') @patch('app.main.replace_skills_with_json') @patch('app.main.replace_experience_with_json') def test_api_post_cv(replace_experience_with_json_mock, replace_skills_with_json_mock, get_session_mock, extract_user_mock, api_cv_post, client, test_input, replace_skills_with_json_error, replace_experience_with_json_error, expected): if replace_skills_with_json_error is not None: replace_skills_with_json_mock.side_effect = replace_skills_with_json_error
def errors(x): raise OperationalError(None, None, None, None)
async def test_database_corruption_while_running(opp, tmpdir, caplog): """Test we can recover from sqlite3 db corruption.""" def _create_tmpdir_for_test_db(): return tmpdir.mkdir("sqlite").join("test.db") test_db_file = await opp.async_add_executor_job(_create_tmpdir_for_test_db) dburl = f"{SQLITE_URL_PREFIX}//{test_db_file}" assert await async_setup_component(opp, DOMAIN, {DOMAIN: { CONF_DB_URL: dburl }}) await opp.async_block_till_done() caplog.clear() opp.states.async_set("test.lost", "on", {}) sqlite3_exception = DatabaseError("statement", {}, []) sqlite3_exception.__cause__ = sqlite3.DatabaseError() with patch.object( opp.data[DATA_INSTANCE].event_session, "close", side_effect=OperationalError("statement", {}, []), ): await async_wait_recording_done_without_instance(opp) await opp.async_add_executor_job(corrupt_db_file, test_db_file) await async_wait_recording_done_without_instance(opp) with patch.object( opp.data[DATA_INSTANCE].event_session, "commit", side_effect=[sqlite3_exception, None], ): # This state will not be recorded because # the database corruption will be discovered # and we will have to rollback to recover opp.states.async_set("test.one", "off", {}) await async_wait_recording_done_without_instance(opp) assert "Unrecoverable sqlite3 database corruption detected" in caplog.text assert "The system will rename the corrupt database file" in caplog.text assert "Connected to recorder database" in caplog.text # This state should go into the new database opp.states.async_set("test.two", "on", {}) await async_wait_recording_done_without_instance(opp) def _get_last_state(): with session_scope(opp=opp) as session: db_states = list(session.query(States)) assert len(db_states) == 1 assert db_states[0].event_id > 0 return db_states[0].to_native() state = await opp.async_add_executor_job(_get_last_state) assert state.entity_id == "test.two" assert state.state == "on" opp.bus.async_fire(EVENT_OPENPEERPOWER_STOP) await opp.async_block_till_done() opp.stop()
def test_function(session): session.execute("select 1;") raise OperationalError(statement=mock.ANY, params=mock.ANY, orig=mock.ANY)
def test_retry_if_data_catalog_exception(self, engine, conn): dialect = engine.dialect exc = OperationalError( "", None, "Database does_not_exist not found. Please check your query.") self.assertFalse( dialect._retry_if_data_catalog_exception(exc, "does_not_exist", "does_not_exist")) self.assertFalse( dialect._retry_if_data_catalog_exception(exc, "does_not_exist", "this_does_not_exist")) self.assertTrue( dialect._retry_if_data_catalog_exception(exc, "this_does_not_exist", "does_not_exist")) self.assertTrue( dialect._retry_if_data_catalog_exception(exc, "this_does_not_exist", "this_does_not_exist")) exc = OperationalError( "", None, "Namespace does_not_exist not found. Please check your query.") self.assertFalse( dialect._retry_if_data_catalog_exception(exc, "does_not_exist", "does_not_exist")) self.assertFalse( dialect._retry_if_data_catalog_exception(exc, "does_not_exist", "this_does_not_exist")) self.assertTrue( dialect._retry_if_data_catalog_exception(exc, "this_does_not_exist", "does_not_exist")) self.assertTrue( dialect._retry_if_data_catalog_exception(exc, "this_does_not_exist", "this_does_not_exist")) exc = OperationalError( "", None, "Table does_not_exist not found. Please check your query.") self.assertFalse( dialect._retry_if_data_catalog_exception(exc, "does_not_exist", "does_not_exist")) self.assertTrue( dialect._retry_if_data_catalog_exception(exc, "does_not_exist", "this_does_not_exist")) self.assertFalse( dialect._retry_if_data_catalog_exception(exc, "this_does_not_exist", "does_not_exist")) self.assertTrue( dialect._retry_if_data_catalog_exception(exc, "this_does_not_exist", "this_does_not_exist")) exc = OperationalError("", None, "foobar.") self.assertTrue( dialect._retry_if_data_catalog_exception(exc, "foobar", "foobar")) exc = ProgrammingError( "", None, "Database does_not_exist not found. Please check your query.") self.assertFalse( dialect._retry_if_data_catalog_exception(exc, "does_not_exist", "does_not_exist")) self.assertFalse( dialect._retry_if_data_catalog_exception(exc, "does_not_exist", "this_does_not_exist")) self.assertFalse( dialect._retry_if_data_catalog_exception(exc, "this_does_not_exist", "does_not_exist")) self.assertFalse( dialect._retry_if_data_catalog_exception(exc, "this_does_not_exist", "this_does_not_exist"))
def raiser(): raise OperationalError("foo", {}, psycopg2.OperationalError())