def _connect(state):
# Open DB connection
state.db_conn = db_iface.db_open(state.model.filepath)
if state.db_conn:
pass
'''
_D.DEBUG(
__name__,
"DB Stats found on disk",
'filepath', state.model.filepath,
'db_conn', state.db_conn
)
'''
else:
state.db_conn = db_iface.db_create(state.model.filepath)
'''
_D.WARNING(
__name__,
"DB Stats not found on disk, creating new",
'filepath', state.model.filepath,
'db_conn', state.db_conn
)
'''
c = state.db_conn.cursor()
sql = "CREATE TABLE sessions_store \
( \
id INTEGER PRIMARY KEY, \
date INTEGER, \
backup_type TEXT, \
size INTEGER \
)"
c.execute(sql)
state.db_conn.commit()
def _connect(state):
# Open DB connection
state.db_conn = db_iface.db_open(state.model.filepath)
if state.db_conn:
pass
'''
_D.DEBUG(
__name__,
"File Versioned Tracking DB found on disk",
'filepath', state.model.filepath,
'db_conn', state.db_conn
)
'''
else:
state.db_conn = db_iface.db_create(state.model.filepath)
'''
_D.DEBUG(
__name__,
"File Versioned Tracking DB not found on disk, creating new",
'filepath', state.model.filepath,
'db_conn', state.db_conn
)
'''
c = state.db_conn.cursor()
sql = "CREATE TABLE versions \
( \
id INTEGER PRIMARY KEY, \
date INTEGER UNIQUE, \
file TEXT, \
size INTEGER \
)"
c.execute(sql)
state.db_conn.commit()
def add_stats(state, date, size):
"""
Add statistical record for session stores:
- date as plain text
- size in bytes, integer
"""
# Open second temporary connection
conn_temp = db_iface.db_open(state.model.filepath)
cursor_temp = conn_temp.cursor()
sql = "INSERT INTO stats \
VALUES ('{}','{}') \
".format(date, size)
cursor_temp.execute(sql)
# Commit and close temporary connection
conn_temp.commit()
conn_temp.close()
def add_update_storage_file(state, filepath, modified, size):
"""
Adds or updates existing storage file with new date/size
"""
# Open second temporary connection, if not opened yet
if not state.db_conn_upload or not state.db_curs_upload:
state.db_conn_upload = db_iface.db_open(state.model.filepath)
state.db_curs_upload = state.db_conn_upload.cursor()
state.db_count_upload = 0
sql = "INSERT OR REPLACE INTO files_storage \
(name, modified, size) VALUES('{}','{}','{}') \
".format(filepath, modified, size)
state.db_curs_upload.execute(sql)
# Commit after 50 queries
state.db_count_upload += 1
if state.db_count_upload > 50:
try:
state.db_conn_upload.commit()
_D.DEBUG(
__name__,
"SQLite3 Commited (groups of 50)",
'last file', filepath
)
except Exception as e:
import sys
(exc_type, exc_value, exc_traceback) = sys.exc_info()
_D.WARNING(
__name__,
"SQLite3 Error",
'msg', exc_value,
'file', filepath
)
raise e
state.db_count_upload = 0
def _connect(state):
# Open DB connection
state.db_conn = db_iface.db_open(state.model.filepath)
if state.db_conn:
pass
'''
_D.DEBUG(
__name__,
"File Incremental Tracking DB found on disk",
'filepath', state.model.filepath,
'db_conn', state.db_conn
)
'''
else:
state.db_conn = db_iface.db_create(state.model.filepath)
'''
_D.DEBUG(
__name__,
"File Incremental Tracking DB not found on disk, creating new",
'filepath', state.model.filepath,
'db_conn', state.db_conn
)
'''
c = state.db_conn.cursor()
# Table files: list of files in storage
sql = "CREATE TABLE files_storage \
( \
name TEXT UNIQUE, \
modified INTEGER, \
size INTEGER \
)"
c.execute(sql)
# Table files_local: list of files found on local disk
sql = "CREATE TABLE files_local \
( \
name TEXT UNIQUE, \
modified INTEGER, \
size INTEGER \
)"
c.execute(sql)
# Table files_upload: list of files to be stored
sql = "CREATE TABLE files_upload \
( \
name TEXT UNIQUE, \
modified INTEGER, \
size INTEGER \
)"
c.execute(sql)
# Table stats: stored amount of data by date
sql = "CREATE TABLE stats \
( \
date TEXT, \
size INTEGER \
)"
c.execute(sql)
# Table params: about this backup
sql = "CREATE TABLE params \
( \
id INTEGER PRIMARY KEY, \
dir_local TEXT, \
url_storage TEXT \
)"
c.execute(sql)
state.db_conn.commit()
def find_local_storage_diff(state, max_total_size):
"""
Finds files that new/modified compared to files already in storage.
maxSize (if > 0) specifies max total size of files.
"""
c = state.db_conn.cursor()
# Open second temporary connection
conn_temp = db_iface.db_open(state.model.filepath)
cursor_temp = conn_temp.cursor()
# How many files are already in storage
sql = "SELECT COUNT(*) FROM files_storage"
c.execute(sql)
row = c.fetchone()
# Storage is empty ?
if not row or row[0] == 0:
# Use all local files
sql = "SELECT DISTINCT * FROM files_local"
else:
# Find difference
sql = "SELECT DISTINCT a.* \
FROM files_local a \
INNER JOIN files_storage b \
ON \
a.name NOT IN (SELECT name FROM files_storage) \
OR \
a.name=b.name AND (a.modified!=b.modified OR a.size!=b.size)"
c.execute(sql)
# For each found file...
i = 0
total_size = 0
for row in c:
total_size += row[2]
if max_total_size <= 0:
add = True
elif total_size < max_total_size or i == 0:
# To make sure first file can bypass max_total_size
add = True
else:
add = False
if add:
sql = "INSERT INTO files_upload \
VALUES ('{}','{}','{}') \
".format(row[0], row[1], row[2])
cursor_temp.execute(sql)
i += 1
else:
# Deduct this file size from total if beyond max
total_size -= row[2]
if total_size < 0:
total_size = 0
# Commit and close temporary connection
if i > 0:
conn_temp.commit()
conn_temp.close()
# Debug
'''
_D.DEBUG(
__name__,
"File Incremental Tracking DB found differences",
'TODO', 'Check count!',
'count', i,
'total_size', fmtutil.byte_size(total_size),
'max_total_size', fmtutil.byte_size(max_total_size),
'% of allowed', 100 * total_size / max_total_size
)
'''
return 0, None