def error_loop(self):
_errors = [
sqlite3.OperationalError("database is locked"),
sqlite3.OperationalError("Not expected")
]
for _err in _errors:
yield _err
def query(self, query_string, max_rows=None):
self._begin()
c = self.db.cursor()
queries = filter(None, [q.strip() for q in query_string.split('\0')])
if not queries:
raise sqlite3.OperationalError('empty query')
desc = None
result = []
for qs in queries:
c.execute(qs)
d = c.description
if d is None:
continue
if desc is None:
desc = d
elif d != desc:
raise sqlite3.OperationalError(
'Multiple incompatible selects in '
'multiple sql-statement query')
if max_rows:
if not result:
result = c.fetchmany(max_rows)
elif len(result) < max_rows:
result = result + c.fetchmany(max_rows - len(result))
else:
result = c.fetchall()
self._finish()
if desc is None:
return (), ()
items = []
for name, type, width, ds, p, scale, null_ok in desc:
if type == 'NUMBER':
if scale == 0:
type = 'i'
else:
type = 'n'
elif type == 'DATE':
type = 'd'
else:
type = 's'
items.append({
'name': name,
'type': type,
'width': width,
'null': null_ok,
})
return items, result
def manage_DataSources(data_dir: str = DEFAULT_DATA_DIR) -> map:
if os.sep not in data_dir:
data_dir = os.path.join(
DEFAULT_DATA_DIR,
data_dir) # create a subdirectory DEFAULT_DATA_DIR
error_msg = ''
if os.path.exists(data_dir):
if not os.path.isdir(data_dir):
error_msg = f"""
The Zope SQLite Database Adapter requires the
existence of the directory, <code>{data_dir}</code>.
This exists, but is not a directory.
"""
else:
if data_dir != DEFAULT_DATA_DIR:
error_msg = f"""
The Zope SQLite Database Adapter requires the
existence of the directory, <code>{data_dir}</code>.
Please create it on the file system.
"""
elif not os.path.exists(DEFAULT_DATA_DIR):
# it's quite safe to create the default path
os.mkdir(DEFAULT_DATA_DIR)
if error_msg:
raise sqlite3.OperationalError(cleandoc(error_msg))
return map(
lambda d: (d, ''),
filter(
lambda f, i=os.path.isfile, d=data_dir, j=os.path.join: i(j(d, f)),
sorted(os.listdir(data_dir))))
def getDefault(tableName, colName) :
"""Specify a table's column and the function returns it's default value."""
cols = photosDict.execute("PRAGMA table_info('%s')" % tableName)
for col in cols :
if col['name'] == colName :
return col['dflt_value']
raise sqlite3.OperationalError('no such column: %s' % colName)
def execute_sqlite3(path, cmds):
"""Execute 'cmds' on SQLite database 'path'"""
import sqlite3
conn = sqlite3.connect(path)
cursor = conn.cursor()
# overwrites deleted content with zeros
# https://www.sqlite.org/pragma.html#pragma_secure_delete
from Options import options
if options.get('shred'):
cursor.execute('PRAGMA secure_delete=ON')
for cmd in cmds.split(';'):
try:
cursor.execute(cmd)
except sqlite3.DatabaseError, exc:
raise sqlite3.DatabaseError('%s: %s' %
(Common.decode_str(exc), path))
except sqlite3.OperationalError, exc:
logger = logging.getLogger(__name__)
if exc.message.find('no such function: ') >= 0:
# fixme: determine why randomblob and zeroblob are not
# available
logger.warning(exc.message)
else:
raise sqlite3.OperationalError('%s: %s' %
(Common.decode_str(exc), path))
def execute_sqlite3(path, cmds):
"""Execute 'cmds' on SQLite database 'path'"""
import sqlite3
import contextlib
with contextlib.closing(sqlite3.connect(path)) as conn:
cursor = conn.cursor()
# overwrites deleted content with zeros
# https://www.sqlite.org/pragma.html#pragma_secure_delete
from bleachbit.Options import options
if options.get('shred'):
cursor.execute('PRAGMA secure_delete=ON')
for cmd in cmds.split(';'):
try:
cursor.execute(cmd)
except sqlite3.OperationalError as exc:
if str(exc).find('no such function: ') >= 0:
# fixme: determine why randomblob and zeroblob are not
# available
logger.exception(exc.message)
else:
raise sqlite3.OperationalError('%s: %s' % (exc, path))
except sqlite3.DatabaseError as exc:
raise sqlite3.DatabaseError('%s: %s' % (exc, path))
cursor.close()
conn.commit()
def verifyTable():
try:
c.execute(
"SELECT name FROM sqlite_master WHERE type='table' and name='users';"
)
if c.fetchall() == []:
raise sqlite3.OperationalError("Table Missing")
c.execute("PRAGMA table_info(users);")
if c.fetchall() != [(0, 'key', 'INTEGER', 0, None, 1),
(1, 'name', 'TEXT', 0, None, 0),
(2, 'password', 'TEXT', 0, None, 0)]:
c.execute("DROP TABLE users;")
raise sqlite3.OperationalError("Malformed Table")
except sqlite3.OperationalError:
print("Password database malformed. Resetting...")
init()
class ConnectionManager(object):
"""
Assumes a database that will work with cursor objects
"""
def __init__(self):
# test out the connection...
conn = sqlite3.connect(db_location)
conn.close()
def query(self, sql, args):
conn = None
retry_count = 0
while not conn and retry_count <= 10:
# If there is trouble reading the file, retry for 10 attempts
# then just give up...
try:
conn = sqlite3.connect(db_location)
except sqlite3.OperationalError, x:
retry_count += 1
time.sleep(0.001)
if not conn and retry_count > 10:
raise sqlite3.OperationalError("Can't connect to sqlite database.")
cursor = conn.cursor()
cursor.execute(sql, args)
res = cursor.fetchall()
conn.close()
return res
def saveData(self, hash, url, content):
if self.conn:
sql = "INSERT INTO Webpage (hash, url, content) VALUES (?, ?, ?);"
self.conn.execute(sql, (hash, url, content))
else:
raise sqlite3.OperationalError(
'Database is not connected. Can not save Data!')
def __init__(self):
self.__logger = Logger()
self._request_exceptions = [type(item) for item in [requests.ConnectionError(), requests.HTTPError(),
requests.TooManyRedirects(), requests.Timeout(),
requests.TooManyRedirects(),
requests.RequestException(), requests.ConnectTimeout(),
requests.ReadTimeout()]]
self._system_errors = [type(item) for item in [KeyError(), AttributeError(), IndexError(),
ZeroDivisionError(), SystemError(), ValueError(),
AssertionError()]]
self._file_errors = [type(item) for item in [FileExistsError(), FileNotFoundError()]]
self._database_errors = [type(item) for item in [sqlite3.Error(), sqlite3.DataError(),
sqlite3.ProgrammingError(), sqlite3.DatabaseError(),
sqlite3.NotSupportedError(), sqlite3.IntegrityError(),
sqlite3.InterfaceError(), sqlite3.InternalError(),
sqlite3.OperationalError()]]
self._speech_recognizer_errors = [type(item) for item in
[sr.RequestError(), sr.UnknownValueError(), sr.WaitTimeoutError(),
sr.RequestError()]]
self.__logger.info('ExceptionsHandler was successfully initialized.', __name__)
def execute_query(cls, sql=None, args=None, limit=None, offset=None):
if isinstance(limit, int) and limit < 1:
raise ValueError('查询条数不能小于1')
if not isinstance(limit, int) and limit:
raise ValueError('参数limit类型非法')
if isinstance(offset, int) and offset < 0:
raise ValueError('跳过条数不能小于0')
if not isinstance(offset, int) and offset:
raise ValueError('参数offset类型非法')
if offset and limit is None:
raise ValueError('跳过条数不为空时，查询条数不能为空')
if limit:
sql = '%s limit %d' % (sql, limit)
if offset:
sql = '%s offset %d' % (sql, offset)
logging.info('[ %s %s ]' % (sql, args))
conn = Connect.connect()
cur = conn.cursor()
try:
cur.execute(sql, args)
except sqlite3.OperationalError as e:
cur.close()
conn.close()
raise sqlite3.OperationalError('[ 数据库执行失败：%s ]' % e)
rest = cur.fetchall()
cur.close()
conn.close()
return rest
async def _mark_finished(self):
try:
await self.db.execute(f"update process set is_processed=True where ID={self.db_record_id}")
await self.db.commit()
except Exception as e:
await self.db.rollback()
raise sqlite3.OperationalError(f'{e.__str__()}, db_path={self.db_path}')
def create_postmortem_database():
# Create the postmortem database
# Create a database connection and a cursor for executing commands.
conn = sqlite3.connect('postmortem.db')
c = conn.cursor()
try:
# Create database with tables
c.executescript('''
CREATE TABLE "Sources" (
"source_id" INTEGER PRIMARY KEY AUTOINCREMENT,
"url" TEXT,
"last_access_timestamp" TEXT,
"http_status_code" INTEGER,
PRIMARY KEY("source_id"));
CREATE TABLE "Postmortems" (
"content" TEXT,
"url" TEXT,
"title" TEXT,
"fk_source_id" INTEGER,
"postmortem_id" INTEGER PRIMARY KEY AUTOINCREMENT,
FOREIGN KEY("fk_source_id") REFERENCES "Sources"("source_id"));
''')
# close the database connection
conn.commit()
except sqlite3.OperationalError as oe:
raise sqlite3.OperationalError(
f'Error occured while creating database: {oe}')
conn.close()
def test_sqlite3_operational_error(mocker):
session = urlquick.Session()
mocked = mocker.patch.object(session.cache_adapter, "conn", autospec=True)
mocked.execute.side_effect = sqlite3.OperationalError("Boom!")
with pytest.raises(sqlite3.OperationalError):
session.cache_adapter.wipe()
def create_db_file(data_dir: str, connection_string: str) -> None:
db_path = os.path.join(data_dir, connection_string)
if os.path.exists(db_path):
return
if data_dir == DEFAULT_DATA_DIR:
if not os.path.exists(DEFAULT_DATA_DIR):
os.mkdir(DEFAULT_DATA_DIR)
if connection_string:
with open(db_path, 'w') as f:
f.write('')
else:
error_msg = f"""The Zope SQLite Database Adapter requires the
existence of the file <code>{connection_string}</code> in
the directory <code>{data_dir}</code>.
For security reasons, you are allowed to create new
databases in the default sqlite directory only:
{DEFAULT_DATA_DIR}
If you want to use a database at a different location,
you have to manage the directory and the database files
on the file system."""
raise sqlite3.OperationalError(cleandoc(error_msg))
def executewait(dbhandle, statement, error, retry=False, wait=5, args=()):
'''execute sql statement.
Retry on error, if retry is True.
Returns a cursor object.
'''
cc = dbhandle.cursor()
i = 20
while i > 0:
try:
cc.execute(statement, args)
return cc
except sqlite3.OperationalError as e:
E.warn("import failed: msg=%s, statement=\n %s" %
(str(e), statement))
# TODO: check for database locked msg
if not retry:
raise e
if not re.search("locked", str(e)):
raise e
time.sleep(wait)
i -= 1
continue
break
raise sqlite3.OperationalError("Database locked and too many retries")
def save(self):
try:
conn = sqlite3.connect(self.db_path)
c = conn.cursor()
c.execute("""INSERT INTO patients (first_name, last_name, birth_date, phone, document_type, document_id)
VALUES ('{}', '{}', '{}', '{}', '{}', '{}');""".format(
self.first_name, self.last_name, self.birth_date, self.phone, self.document_type, self.document_id
))
conn.commit()
c.close()
except OSError:
raise OSError("Ошибка системы: Данные не сохранены")
except UnicodeError:
raise UnicodeError("Ошибка кодировки: Данные не сохранены")
except RuntimeError:
raise RuntimeError("RuntimeError: Данные не сохранены")
except sqlite3.OperationalError:
raise sqlite3.OperationalError()
except sqlite3.IntegrityError:
raise sqlite3.IntegrityError('sqlite3.IntegrityError: уже есть такой номер документа')
except FileExistsError:
raise FileExistsError
except FileNotFoundError:
raise FileNotFoundError
except IsADirectoryError:
raise IsADirectoryError
except PermissionError:
raise PermissionError
def main(args):
if not args:
usage()
raise Exception("No filename given")
filename = args[0]
sql_src_dir = os.path.dirname(os.path.realpath(__file__))
dest_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..",
"var", filename)
if not os.path.isdir(os.path.dirname(dest_file)):
os.makedirs(os.path.dirname(dest_file))
try:
dbconn = sqlite3.connect(dest_file)
except sqlite3.OperationalError as e:
raise sqlite3.OperationalError(str(e) + " file: {0}".format(dest_file))
schema_file = os.path.join(sql_src_dir, "fattybrewery-schema.sql")
init_file = os.path.join(sql_src_dir, "fattybrewery-init.sql")
for f in [schema_file, init_file]:
if not os.path.isfile(f):
raise OSError("Unable to find required file {0}".format(f))
for f in [schema_file, init_file]:
with open(f) as sfh:
sql = sfh.read()
dbconn.executescript(sql)
dbconn.close()
def saveData(self, url, pageSource, keyword=''):
if self.conn:
sql = '''INSERT INTO Webpage (url, pageSource, keyword) VALUES (?, ?, ?);'''
self.conn.execute(sql, (url, pageSource, keyword))
else:
raise sqlite3.OperationalError(
'Database is not connected. Can not save Data!')
def create_probe_database(log, db, organisms, exons):
"""Create the exon-match database"""
log.info("Creating the exon-match database")
conn = sqlite3.connect(db)
c = conn.cursor()
c.execute("PRAGMA foreign_keys = ON")
try:
create_string = [org + ' text' for org in organisms]
query = "CREATE TABLE matches (exon text primary key, {0})".format(
','.join(create_string))
c.execute(query)
query = "CREATE TABLE match_map (exon text primary key, {0})".format(
','.join(create_string))
c.execute(query)
# convert exons to list of tuples for executemany
all_exons = [(exon, ) for exon in exons]
c.executemany("INSERT INTO matches(exon) values (?)", all_exons)
c.executemany("INSERT INTO match_map(exon) values (?)", all_exons)
except sqlite3.OperationalError, e:
log.critical("Database already exists")
if e[0] == 'table matches already exists':
answer = raw_input("Database already exists. Overwrite [Y/n]? ")
if answer == "Y" or "YES":
os.remove(db)
conn, c = create_probe_database(db, organisms, exons)
else:
sys.exit(2)
else:
log.critical("Cannot create database")
raise sqlite3.OperationalError("Cannot create database")
def check_if_table_in_db(self):
self.cursor.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name=?;",
(self._db_table, ),
)
if self.cursor.fetchone() is None:
raise sqlite3.OperationalError(f"no such table: {self._db_table}")
def execute(self, query, *args):
"""
Execute query
:param query: Query string.
:param args: Query values.
"""
n_attempts = 0
while n_attempts <= self.max_attempts:
try:
if args:
self.__cur.execute(query, *args)
else:
self.__cur.execute(query)
break
except sqlite3.OperationalError as e:
error_text = str(e)
n_attempts += 1
except Exception as e:
raise Exception(str(e))
if n_attempts > self.max_attempts:
raise sqlite3.OperationalError(
"Maximum attempts exceeded for sqlite3 execute: {0}".
format(error_text))
def execute_query(self):
"""
Executing a query on click Execute button
:return: void
"""
try:
self.conn.row_factory = sqlite3.Row
cursor = self.conn.cursor()
if not len(self.query.toPlainText()):
raise sqlite3.OperationalError("Can\'t execute empty query")
cursor.execute(self.query.toPlainText())
if cursor.description:
model = TableModel(None, cursor)
self.results.setModel(model)
except sqlite3.OperationalError as e:
QMessageBox.critical(
self, 'Operational Error', '%s. %s' %
(e.__str__(),
"Please check the documentation https://www.sqlite.org/lang.html to correct the query."
))
except Exception as e:
QMessageBox.critical(self, 'Error', e.__str__())
raise e
else:
QMessageBox.information(self, 'Executed', 'It\'s Ok. Done!')
def execute(self, operation, parameters=()):
self.connection.test_data["execute"].append((operation, parameters))
operation, types_ = google.cloud.bigquery.dbapi.cursor._extract_types(
operation)
if parameters:
operation, parameters = self.__convert_params(
operation, parameters)
else:
operation = operation.replace("%%", "%")
operation = self.__handle_comments(operation)
operation = self.__handle_array_types(operation)
operation = self.__handle_problematic_literal_inserts(operation)
operation = self.__handle_unnest(operation)
operation = self.__handle_true_false(operation)
operation = operation.replace(" UNION DISTINCT ", " UNION ")
if operation:
try:
self.cursor.execute(operation, parameters)
except sqlite3.OperationalError as e: # pragma: NO COVER
# Help diagnose errors that shouldn't happen.
# When they do, it's likely due to sqlite versions (environment).
raise sqlite3.OperationalError(*((operation, ) + e.args +
(sqlite3.sqlite_version, )))
self.description = self.cursor.description
self.rowcount = self.cursor.rowcount
def query(self, sql, params=None):
"""
Query the database using the supplied SQL. The SQL should follow the
formatting rules defined in :pep:`249` for SQLite, whos `paramstyle` is
`qmark`. Doing so will protect against SQL injection attacks.
For example::
sql = "SELECT * FROM ZipCodes WHERE zip=?"
params = ('94949',)
"""
conn = None
retry_count = 0
while not conn and retry_count <= 10:
# If there is trouble reading the file, retry for 10 attempts
# then just give up...
try:
conn = sqlite3.connect(db_location)
except sqlite3.OperationalError:
retry_count += 1
time.sleep(0.001)
if not conn and retry_count > 10:
raise sqlite3.OperationalError(
"Can't connect to sqlite database: '%s'." % db_location)
cursor = conn.cursor()
if params is not None:
cursor.execute(sql, params)
else:
cursor.execute(sql)
res = cursor.fetchall()
conn.close()
return res
def __getitem__(self, item):
if item in self.__cache:
return self.__cache[item]
failed = 0
while True:
try:
res = self.__cursor.execute("SELECT json from genes where gid=?", (item,)).fetchmany()
break
except sqlite3.OperationalError as exc:
failed += 1
if failed > 10:
raise sqlite3.OperationalError(exc)
sleep(3)
if res:
try:
gene = self.__load_gene(res[0][0])
self.__cache[item] = gene
return gene
except IndexError:
raise IndexError(res)
else:
return None
def create_table_for_img_info(dbcr, img_info):
'Creates a database table, in a database cursor, to store for the input img_info'
create_command = 'CREATE TABLE {}({} TEXT PRIMARY KEY,'.format(
SQLITE_IMG_INFO_TABLE, SQLITE_IMG_INFO_FNAME)
for key in list(img_info.keys()):
create_command += ' "{}" TEXT,'.format(info_key_to_db_name(key))
create_command = create_command[0:-1] + ')'
# Can make a rare race condition if multiple processes try to create the file at the same time;
# If that happens, the error is:
# sqlite3.OperationalError: table img_info already exists for file .......
try:
dbcr.execute(create_command)
except sqlite3.OperationalError as op_err:
if 'table {} already exists'.format(
SQLITE_IMG_INFO_TABLE) in op_err.message:
# another process has just created the table, so sleep(1)
# This is only when a db file is created so isn't called often
# (and the race condition is rare!)
time.sleep(1)
else:
# everything else needs to be reported and raised immediately:
raise sqlite3.OperationalError(op_err.message)
except sqlite3.Error as sq_err:
# everything else needs to be reported and raised immediately:
raise sqlite3.Error(sq_err.message)
def _new_database(self):
'''
create and connect to a new sqlite database.
raise an error if there already is a database in place, asking the
user to manually remove the database (for safety reasons)
'''
# TODO: remove next two lines after testing -> don't automatically remove
if os.path.exists(self.database):
os.remove(self.database)
if os.path.exists(self.database):
message = ('Database already exists, please remove manually: %s'
%self.database)
logger.error(message)
raise IOError(message)
else:
logger.info('Database not found, creating database %s' %self.database)
try:
self.connection = sqlite3.connect(
self.database,
detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES)
except:
message = 'Failed to create database: %s' %self.database
logger.error(message)
raise sqlite3.OperationalError(message) # re-raise error
self._create_dbstructure()
sqlite3.register_adapter(bool, int)
sqlite3.register_converter("BOOLEAN", lambda v: bool(int(v)))
# tuples
self.connection.row_factory = sqlite3.Row
def savedb(self,Db=None,overwrite=False):
"""Store the table in a db
Parameters:
1. Db: the sqlite3 db file to open
2. overwrite: a boolean switch.
False: do not overwrite if there is a table with the same name already
True: overwrite if there is a table with the same name already
"""
# if table is empty, raise an exception
if self.isempty(): raise Exception('Empty table')
# connect to the local database file
dbconnect=sqlite3.connect(Db)
cursor=dbconnect.cursor()
tablename=self.get_name()
# create a table
sqlcmd="""create table %s (""" %(tablename)+','.join(self.columns)+""")"""
try:
cursor.execute(sqlcmd)
except sqlite3.OperationalError,e:
if overwrite:
warnings.warn("Overwritting the existing table")
cursor.execute("""drop table %s""" %(tablename))
cursor.execute(sqlcmd)
else:
raise sqlite3.OperationalError(e)
def test_no_fai(self):
with tempfile.NamedTemporaryFile(suffix="db") as db_handle:
db = db_handle.name
genome_file = tempfile.NamedTemporaryFile("wb", suffix=".fa.gz", prefix="Chr5", dir=".")
jconf = self.configuration.copy()
jconf.db_settings.db = db
jconf.reference.genome_fai = None
with resource_stream("Mikado.tests", "chr5.fas.gz") as _:
genome_file.write(_.read())
genome_file.flush()
jconf.reference.genome = genome_file.name
logger = utilities.log_utils.create_default_logger("test_no_fai", "DEBUG")
with self.assertLogs("test_no_fai", level="DEBUG") as cmo:
seri = serializers.junction.JunctionSerializer(
self.junction_file,
configuration=jconf,
logger=logger
)
self.assertEqual(seri.db_settings.db, db)
seri()
genome_file.close()
# Now check that there are junctions in the temp database
with sqlite3.connect(db) as conn:
try:
result = conn.execute("select count(*) from junctions").fetchone()[0]
except sqlite3.OperationalError:
msg = "Failed to obtain data from {}".format(db)
raise sqlite3.OperationalError([msg] + cmo.output)
self.assertGreater(result, 0)
if os.path.exists("{}.fai".format(genome_file.name)):
os.remove("{}.fai".format(genome_file.name))
