def create_tables():
try:
SiteToSearch.create_table(safe=False, )
# SiteToSearch.get_or_create(url="https://www.terra.com.br/")
# SiteToSearch.get_or_create(url="https://www.r7.com/")
SiteToSearch.get_or_create(url="http://g1.globo.com/")
UrlFound.create_table(safe=False)
Page.create_table(safe=False)
PageElement.create_table(safe=False)
print("Tabelas criadas com sucesso!\n")
print(get_table_name(SiteToSearch()))
print(get_table_name(UrlFound()))
print(get_table_name(Page()))
print(get_table_name(PageElement()))
return True
except ProgrammingError:
print("A tabela já existe. Dropando tabelas\n")
delete_tables()
except OperationalError:
print("Ocorreu algum erro ou há alguma tabela já existente!")
print(OperationalError.with_traceback())
print(OperationalError.mro())
return False
def test_cart_orm_db_setup_error(self, mock_cart_dbcon, mock_file_dbcon):
"""Call database_setup."""
mock_cart_dbcon.side_effect = OperationalError('Failing')
mock_file_dbcon.side_effect = OperationalError('Failing')
hit_exception = False
os.environ['DATABASE_CONNECT_ATTEMPTS'] = '3'
try:
OrmSync.dbconn_blocking()
except OperationalError:
hit_exception = True
self.assertTrue(hit_exception)
del os.environ['DATABASE_CONNECT_ATTEMPTS']
async def connect(self, safe=True):
if self.deferred:
raise OperationalError('Database has not been initialized')
if not self.closed:
if safe:
return
raise OperationalError('Connection already open')
with self.exception_wrapper:
self.pool = await self._connect(self.database,
**self.connect_kwargs)
self.closed = False
def test_sql_error_reconnect(self):
class RetryDatabase(SqliteDatabase):
def sql_error_handler(self, exc, sql, params, require_commit):
if isinstance(exc, OperationalError):
self.close()
self.connect()
return True
db = RetryDatabase(':memory:', autocommit=False)
conn1 = mock.Mock(name='conn1')
conn2 = mock.Mock(name='conn2')
curs = mock.Mock(name='curs')
curs.execute.side_effect = OperationalError()
with mock.patch.object(db, '_connect') as pc:
pc.side_effect = [conn1, conn2]
with mock.patch.object(db, 'get_cursor') as pgc:
pgc.return_value = curs
db.connect()
self.assertRaises(OperationalError, db.execute_sql,
'fake query')
conn1.close.assert_called_once_with()
pgc.assert_called_once_with()
self.assertTrue(db.get_conn() is conn2)
def main(*argv):
"""Main method to start the httpd server."""
parser = ArgumentParser(description='Run the uniqueid server.')
parser.add_argument('-c', '--config', metavar='CONFIG', type=str,
default=CONFIG_FILE, dest='config',
help='cart config file')
parser.add_argument('--cpconfig', metavar='CONFIG', type=str,
default=CHERRYPY_CONFIG, dest='cpconfig',
help='cherrypy config file')
parser.add_argument('-p', '--port', metavar='PORT', type=int,
default=8051, dest='port',
help='port to listen on')
parser.add_argument('-a', '--address', metavar='ADDRESS',
default='localhost', dest='address',
help='address to listen on')
parser.add_argument('--stop-after-a-moment', help=SUPPRESS,
default=False, dest='stop_later',
action='store_true')
if not argv: # pragma: no cover
argv = sys_argv[1:]
args = parser.parse_args(argv)
OrmSync.dbconn_blocking()
if not UniqueIndexSystem.is_safe():
raise OperationalError('Database version too old {} update to {}'.format(
'{}.{}'.format(*(UniqueIndexSystem.get_version())),
'{}.{}'.format(SCHEMA_MAJOR, SCHEMA_MINOR)
))
stop_later(args.stop_later)
cherrypy.config.update({'error_page.default': error_page_default})
cherrypy.config.update({
'server.socket_host': args.address,
'server.socket_port': args.port
})
cherrypy.quickstart(Root(), '/', args.cpconfig)
def validate_postgres_precondition(driver):
cursor = driver.execute_sql("SELECT extname FROM pg_extension",
("public", ))
if 'pg_trgm' not in [extname for extname, in cursor.fetchall()]:
raise OperationalError("""
"pg_trgm" extension does not exists in the database.
Please run `CREATE EXTENSION IF NOT EXISTS pg_trgm;` as superuser on this database.
""")
def get_conn(self):
if self.closed:
raise OperationalError('Database pool has not been initialized')
return AioConnection(self.pool.acquire(),
autocommit=self.autocommit,
autorollback=self.autorollback,
exception_wrapper=self.exception_wrapper)
def connect_and_check(cls):
"""Connect and check the version."""
cls.dbconn_blocking()
if not MetadataSystem.is_safe(): # pragma: no cover the raise prevents coverage
cls.close()
raise OperationalError('Database version too old {} update to {}'.format(
'{}.{}'.format(*(MetadataSystem.get_version())),
'{}.{}'.format(SCHEMA_MAJOR, SCHEMA_MINOR)
))
def test_db_connect_and_fail(self, test_patch):
"""Try to connect to a database and fail."""
test_patch.side_effect = OperationalError('connection refused')
os.environ['DATABASE_CONNECT_ATTEMPTS'] = '1'
os.environ['DATABASE_CONNECT_WAIT'] = '1'
hit_exception = False
try:
OrmSync.dbconn_blocking()
except OperationalError:
hit_exception = True
self.assertTrue(hit_exception)
async def connect(self, loop=None):
if self.deferred:
raise OperationalError('Database has not been initialized')
#if not self._closed:
#if self._conn_pool is not None:
# raise OperationalError('Connection already open')
self._conn_pool = await self._create_connection(loop=loop)
self._closed = False
with self.exception_wrapper:
self.initialize_connection(self._conn_pool)
def dbconn_blocking():
"""Wait for the db connection."""
dbcon_attempts = get_config().getint('database', 'connect_attempts')
dbcon_wait = get_config().getint('database', 'connect_wait')
while dbcon_attempts:
try:
UniqueIndex.database_connect()
return
except OperationalError:
# couldnt connect, potentially wait and try again
sleep(dbcon_wait)
dbcon_attempts -= 1
raise OperationalError('Failed database connect retry.')
def dbconn_blocking():
"""Wait for the db connection."""
dbcon_attempts = get_config().getint('database', 'connect_attempts')
dbcon_wait = get_config().getint('database', 'connect_wait')
while dbcon_attempts:
try: # pragma: no cover this block doesn't complete
if not DB.is_closed():
DB.close()
DB.connect()
return
except OperationalError:
# couldnt connect, potentially wait and try again
sleep(dbcon_wait)
dbcon_attempts -= 1
raise OperationalError('Failed database connect retry.')
def operation():
raise OperationalError('database is locked')
def operation():
if stats.values.get('database/locked_retries', 0) < 5:
raise OperationalError('database is locked')
return 42
def mock_db_connect(*args, **kwargs):
raise OperationalError()
def fake_database_connect(cls):
"""throw error during connect"""
cls.throw_error = True
raise OperationalError('Failing')
