def storeInDatabase(cur, image_name, data):
cur.execute("INSERT INTO map (name, image_file) values (?, ?) ",
(image_name, sqlite.Binary(data)))
con.commit()
def rebuild_webui(verbose, webui_db_name):
try:
# Check which database name to use
if not webui_db_name or webui_db_name == "":
webui_db_name = get_lib_db_path()
# Connect to the database
db_conn = sqlite3.connect(webui_db_name)
try:
log_message(
"Creating the documentation library webui structure...")
execute_sql_script(db_conn, './mysqldoclib_webui.sql')
c = db_conn.cursor()
try:
for path, dirs, files in os.walk("webui"):
for name in files:
if name.endswith(".html") or name.endswith(
".wbp") or name.endswith(".css"):
# Open HTML file as a utf-8 file
html_string = read_file_content(
os.path.join(path, name))
if not html_string:
continue
log_message("Path: %s, File: %s" % (path, name))
c.execute(
"INSERT OR REPLACE INTO web_object(path, content_type, content, allow_embedded_code_execution) VALUES(?, ?, ?, ?)",
[
"/" + path + "/" + os.path.basename(name),
"text/html", html_string, 1
])
else:
content_type = ""
if name.endswith(".png"):
content_type = "image/png"
elif name.endswith(".gif"):
content_type = "image/gif"
elif name.endswith(".ico"):
content_type = "image/vnd.microsoft.icon"
img_file = open(os.path.join(path, name), "rb")
log_message("Path: %s, File: %s" % (path, name))
c.execute(
"INSERT OR REPLACE INTO web_object(path, content_type, content, allow_embedded_code_execution) VALUES(?, ?, ?, ?)",
[
"/" + path + "/" + os.path.basename(name),
content_type,
sqlite3.Binary(img_file.read()), 0
])
finally:
c.close()
db_conn.commit()
finally:
db_conn.close()
except Exception, e:
log_error(
"An error occurred while opening the database connection. %r" % e)
def CheckBinary(self):
b = sqlite.Binary(chr(0) + "'")
def cache_pages(db_conn, manual_ids):
try:
# Get database cursor
c = db_conn.cursor()
try:
# Get all available manuals versions
c.execute("""-- Select all manuals
SELECT m.id_manual, m.directory, m.description
FROM manual m
ORDER BY m.id_manual""")
rows = c.fetchall()
except Exception, e:
log_error(
"An error occurred while executing the SQL commands. %r" % e)
finally:
c.close()
# Loop over all manuals and cache the contents of the file directory
for id_manual, directory, description in rows:
# if the number of manuals has been limited
if manual_ids:
# only include the given manual
if not str(id_manual) in manual_ids:
log_message("Skipping manual %s." % description)
continue
zip_file = directory + ".zip"
# Locate the zip file, first in the user app dir
# zip_file_path = os.path.join(os.path.join(get_app_data_dir(), 'repository'), zip_file)
# if not os.path.exists(zip_file_path):
# then in the ./repository dir
zip_file_path = os.path.join(os.path.join('.', 'repository'),
zip_file)
if not os.path.exists(zip_file_path):
log_error("The zip file %s cannot be found." % zip_file_path)
continue
log_message("Processing %s ..." % zip_file_path)
lib_zip_file = zipfile.ZipFile(zip_file_path, 'r')
try:
#path = os.path.join('./', directory)
#files = [file for file in os.listdir(path) if file.lower().endswith(".html")]
files = [
file for file in lib_zip_file.namelist()
if file.lower().endswith(".html")
]
file_count = len(files)
file_nr = 0
log_message("Caching manual %s, processing %d file(s) ..." %
(description, file_count))
# Generate synchronization objects
filename_queue = Queue.Queue()
manual_page_data_queue = Queue.Queue()
# Fill filename queue
for f in files: #[:1]:
filename_queue.put(f)
time_start = datetime.datetime.now()
# Start threads
Pool = []
for i in range(1):
thread = ScanManualPageThread()
thread.filename_queue = filename_queue
thread.manual_page_data_queue = manual_page_data_queue
#thread.path = path
thread.lib_zip_file = lib_zip_file
Pool.append(thread)
thread.start()
# Wait for threads to complete
while Pool:
# Process all objects in queue
file_nr += process_manual_page_data_queue(
db_conn, lib_zip_file, directory, id_manual, file_nr,
file_count, manual_page_data_queue)
# Check if there are still threads that are alive
for index, the_thread in enumerate(Pool):
if the_thread.isAlive():
continue
else:
del Pool[index]
break
# Process all objects still left in queue after the threads have all been closed
file_nr += process_manual_page_data_queue(
db_conn, lib_zip_file, directory, id_manual, file_nr,
file_count, manual_page_data_queue)
# Get database cursor
c = db_conn.cursor()
try:
# Update manual to be installed
generation_date = datetime.datetime.now().strftime(
"%Y-%m-%d")
c.execute(
"UPDATE manual SET installed=1, generation_date=? WHERE id_manual=?",
(generation_date, id_manual))
except Exception, e:
log_error(
"An error occurred while updating the manual entry. %r"
% e)
finally:
c.close()
db_conn.commit()
time_duration = datetime.datetime.now() - time_start
log_message(
"%d file(s) of %d processed. Duration %d.%d seconds." %
(file_nr, file_count, time_duration.seconds,
time_duration.microseconds))
# Add the images as web_objects
files = [
file for file in lib_zip_file.namelist()
if file.lower().endswith(".png")
]
log_message("Processing %d image file(s) ..." % len(files))
for filename in files:
try:
# Get database cursor
c = db_conn.cursor()
try:
image_file = lib_zip_file.open(filename)
image_file_string = image_file.read()
if not image_file_string:
continue
# Insert HTML page
c.execute(
"INSERT OR REPLACE INTO web_object(path, content_type, content, allow_embedded_code_execution) VALUES(?, ?, ?, ?)",
[
"/" + directory + "/images/" +
os.path.basename(filename), "image/png",
sqlite3.Binary(image_file_string), 0
])
except Exception, e:
log_error(
"An error occurred while inserting the image file %s. %r"
% (filename, e))
except Exception, e:
log_error(
"An error occurred aquiring a database cursor. %r"
% e)
db_conn.commit()
def _encode(self, item):
return sqlite.Binary(pickle.dumps(item, -1))
def _adapt_geometry(geometry):
"""Return the SQLite BLOB-Geometry representation of a Geometry instance."""
return dbapi2.Binary(blobgeometry.encode_blob_geometry(geometry))
def writeData(self, mapped, value):
cursor, key = mapped
# compress(dumps(value,2))
cursor.execute('insert or replace into %s values (?,?)' % self.table,
(key, sqlite3.Binary(compress(dumps(value, 2)))))
def _encode_items(self, a , b):
return sqlite.Binary(pickle.dumps(a, -1)), sqlite.Binary(pickle.dumps(b, -1))