Exemple #1
0
def getOverviewData(req):
    conn = getConnection()
    num = {}
    num['nodes'] = str(conn.runQuery("select count(*) from node")[0][0])
    num['metadata'] = str(conn.runQuery("select count(*) from nodeattribute")[0][0])
    num['files'] = str(conn.runQuery("select count(*) from nodefile where type='document' or type='image'")[0][0])
    num['size'] = format_filesize(getTotalSize(conn))
    return num
Exemple #2
0
def flush():
    global childids_cache, parentids_cache, _root, db
    tree_lock.acquire()
    try:
        childids_cache = None
        _get_node.cache_clear()
        parentids_cache = None
        db = database.getConnection()
        _root = None
    finally:
        tree_lock.release()
Exemple #3
0
def flush():
    global childids_cache, parentids_cache, _root, db
    tree_lock.acquire()
    try:
        childids_cache = None
        _get_node.cache_clear()
        parentids_cache = None
        db = database.getConnection()
        _root = None
    finally:
        tree_lock.release()
Exemple #4
0
def getOverviewData(req):
    conn = getConnection()
    num = {}
    num['nodes'] = str(conn.runQuery("select count(*) from node")[0][0])
    num['metadata'] = str(
        conn.runQuery("select count(*) from nodeattribute")[0][0])
    num['files'] = str(
        conn.runQuery(
            "select count(*) from nodefile where type='document' or type='image'"
        )[0][0])
    num['size'] = format_filesize(getTotalSize(conn))
    return num
Exemple #5
0
import sys
sys.path += ["../", "."]
import time

from core.init import basic_init
basic_init()

import core.tree as tree
from core.stats import buildStat
from core.db import database

"""
if parameter is given it will be used as period, format: %Y-%m
if not given, current month will be used as period
"""

db = database.getConnection()
args = sys.argv
period = time.strftime("%Y-%m")
fname = None

if len(args) == 2:  # period given
    period = args[1]

if len(args) == 3:  # period and filename given
    period = args[1]
    fname = args[2]

for id in db.runQuery("select id from node where type='collections' or type='collection'"):
    buildStat(tree.getNode(id[0]), period, fname)
Exemple #6
0
def initialize(load=1):
    global db, _root, testmode
    testmode = config.get("host.type", "") == "testing"
    db = database.getConnection()
    db.applyPatches()
    if load:
        getRoot()
    global schema, subnodes, searchParser, searcher
    import schema.schema as schema
    from core.search.ftsquery import DBTYPE
    schema = schema

    if config.get("config.searcher", "") == "fts3":  # use fts3

        from core.search.ftsquery import subnodes, ftsSearcher
        from core.search.ftsparser import ftsSearchParser

        subnodes = subnodes
        searchParser = ftsSearchParser
        searcher = ftsSearcher

        def getTablenames(searcher, schema, db_type):
            tablenames = []
            res = searcher.execute("SELECT * FROM sqlite_master WHERE type='table'", schema, db_type)
            if res is not None:
                tablenames += [t[1] for t in res]
            return tablenames

        # check fts database for tables
        msg = "looking for tables in sqlite database of the searcher ..."
        log.info(msg)
        for schema in searcher.schemas:
            for db_type in list(set(searcher.connames[DBTYPE].values())):
                try:
                    tablenames = getTablenames(searcher, schema, db_type)
                    if tablenames:
                        msg = "found %d tables in sqlite database of the searcher: %r" % (len(tablenames), tablenames)
                        log.info(msg)
                        print "fts3 searcher initialized"
                    else:
                        msg = "found no tables in sqlite database of the searcher ... trying to initialize database"
                        log.warning(msg)
                        searcher.initIndexer(option="init")
                        tablenames = getTablenames(searcher, schema, db_type)
                        if tablenames:
                            msg = "found %d tables in newly initialized sqlite database of the searcher: %r" % (len(tablenames), tablenames)
                            log.info(msg)
                            print "fts3 searcher initialized"
                        else:
                            raise
                except:
                    msg = "could not query tables from fts sqlite database ... searcher may not be functional"
                    log.error(msg)

    else:  # use magpy
        print "magpy searcher initialized"
        from core.search.query import subnodes, mgSearcher
        from core.search.parser import searchParser

        subnodes = subnodes
        searchParser = searchParser
        searcher = mgSearcher

    # load char replacement table

    file = None
    sections = ["chars", "words"]
    data = {"chars": [], "words": []}
    for f in getRoot().getFiles():
        if f.retrieveFile().endswith("searchconfig.txt"):
            file = f
            break

    if file and os.path.exists(file.retrieveFile()):
        section = ""
        for line in open(file.retrieveFile(), "r"):
            line = line[:-1]
            if line.startswith("[") and line.endswith("]"):
                section = line[1:-1]
                continue
            if section in sections:
                data[section].append(line.split("="))
    import utils.utils
    utils.utils.normalization_items = data
Exemple #7
0
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""
import sys
sys.path += ["."]

import core
import logging
import core.config as config
import os
from core.db import database

db = database.getConnection()

write = 1  # parameter: 0=simulate operation, 1=write operation

rootid = None
used_ids = {}
nodemap = {}

# ------- NODES -------

print "Checking", db.runQuery("select count(*) from node")[0][0], "nodes"

res = db.runQuery("select id,name,type from node")
for id, name, type in res:
    used_ids[id] = None
    nodemap[id] = []
Exemple #8
0
def initialize():
    global conn
    conn = database.getConnection()
Exemple #9
0
def initialize(load=1):
    global db, _root, testmode
    testmode = config.get("host.type", "") == "testing"
    db = database.getConnection()
    db.applyPatches()
    if load:
        getRoot()
    global schema, subnodes, searchParser, searcher
    import schema.schema as schema
    from core.search.ftsquery import DBTYPE
    schema = schema

    if config.get("config.searcher", "") == "fts3":  # use fts3

        from core.search.ftsquery import subnodes, ftsSearcher
        from core.search.ftsparser import ftsSearchParser

        subnodes = subnodes
        searchParser = ftsSearchParser
        searcher = ftsSearcher

        def getTablenames(searcher, schema, db_type):
            tablenames = []
            res = searcher.execute(
                "SELECT * FROM sqlite_master WHERE type='table'", schema,
                db_type)
            if res is not None:
                tablenames += [t[1] for t in res]
            return tablenames

        # check fts database for tables
        msg = "looking for tables in sqlite database of the searcher ..."
        log.info(msg)
        for schema in searcher.schemas:
            for db_type in list(set(searcher.connames[DBTYPE].values())):
                try:
                    tablenames = getTablenames(searcher, schema, db_type)
                    if tablenames:
                        msg = "found %d tables in sqlite database of the searcher: %r" % (
                            len(tablenames), tablenames)
                        log.info(msg)
                        print "fts3 searcher initialized"
                    else:
                        msg = "found no tables in sqlite database of the searcher ... trying to initialize database"
                        log.warning(msg)
                        searcher.initIndexer(option="init")
                        tablenames = getTablenames(searcher, schema, db_type)
                        if tablenames:
                            msg = "found %d tables in newly initialized sqlite database of the searcher: %r" % (
                                len(tablenames), tablenames)
                            log.info(msg)
                            print "fts3 searcher initialized"
                        else:
                            raise
                except:
                    msg = "could not query tables from fts sqlite database ... searcher may not be functional"
                    log.error(msg)

    else:  # use magpy
        print "magpy searcher initialized"
        from core.search.query import subnodes, mgSearcher
        from core.search.parser import searchParser

        subnodes = subnodes
        searchParser = searchParser
        searcher = mgSearcher

    # load char replacement table

    file = None
    sections = ["chars", "words"]
    data = {"chars": [], "words": []}
    for f in getRoot().getFiles():
        if f.retrieveFile().endswith("searchconfig.txt"):
            file = f
            break

    if file and os.path.exists(file.retrieveFile()):
        section = ""
        for line in open(file.retrieveFile(), "r"):
            line = line[:-1]
            if line.startswith("[") and line.endswith("]"):
                section = line[1:-1]
                continue
            if section in sections:
                data[section].append(line.split("="))
    import utils.utils
    utils.utils.normalization_items = data