def main(host, port, unix=None, days=1, username=None, password=None,
realm=None, blob_dir=None, storage='1'):
if unix is not None:
addr = unix
else:
if host is None:
host = socket.gethostname()
addr = host, int(port)
wait = True
cs = None
try:
if HAS_BLOB:
cs = ClientStorage(
addr, storage=storage, wait=wait, read_only=True,
username=username, password=password, realm=realm,
blob_dir=blob_dir
)
else:
cs = ClientStorage(
addr, storage=storage, wait=wait, read_only=True,
username=username, password=password, realm=realm
)
cs.pack(wait=wait, days=int(days))
finally:
if cs is not None:
cs.close()
def _main(host, port, unix=None, days=1, username=None, password=None,
realm=None, blob_dir=None, storage='1', shared_blob_dir=True):
if unix is not None:
addr = unix
else:
if host is None:
host = socket.gethostname()
addr = host, int(port)
if blob_dir:
blob_dir = os.path.abspath(blob_dir)
# We do not want to wait until a zeoserver is up and running; it
# should already be running.
wait = False
cs = None
try:
cs = ClientStorage(
addr, storage=storage, wait=wait, read_only=True,
username=username, password=password, realm=realm,
blob_dir=blob_dir, shared_blob_dir=shared_blob_dir,
)
try:
cs.pack(wait=wait, days=int(days))
except ClientDisconnected:
logger = logging.getLogger(__name__)
logger.error("Could not connect to zeoserver. Please make sure it "
"is running.")
sys.exit(1)
finally:
if cs is not None:
cs.close()
def _main(host,
port,
unix=None,
days=1,
username=None,
password=None,
realm=None,
blob_dir=None,
storage='1',
shared_blob_dir=True):
if unix is not None:
addr = unix
else:
if host is None:
host = socket.gethostname()
addr = host, int(port)
if blob_dir:
blob_dir = os.path.abspath(blob_dir)
cs = None
logger = logging.getLogger(__name__)
try:
# We do not want to wait until a zeoserver is up and running; it
# should already be running, so wait=False
cs = ClientStorage(
addr,
storage=storage,
wait=False,
read_only=True,
username=username,
password=password,
realm=realm,
blob_dir=blob_dir,
shared_blob_dir=shared_blob_dir,
)
for i in range(60):
if cs.is_connected():
break
time.sleep(1)
else:
logger.error("Could not connect to zeoserver. Please make sure it "
"is running.")
cs.close()
sys.exit(1)
try:
# The script should not exit until the packing is done.
# => wait=True
cs.pack(wait=True, days=int(days))
except ClientDisconnected:
logger.error("Disconnected from zeoserver. Please make sure it "
"is still running.")
sys.exit(1)
finally:
if cs is not None:
cs.close()
def pack1(addr, storage, days, wait):
cs = ClientStorage(addr, storage=storage, wait_for_server_on_startup=wait)
if wait:
# _startup() is an artifact of the way ZEO 1.0 works. The
# ClientStorage doesn't get fully initialized until registerDB()
# is called. The only thing we care about, though, is that
# registerDB() calls _startup().
cs._startup()
else:
connect(cs)
cs.invalidator = None
cs.pack(wait=1, days=days)
cs.close()
def pack(addr, storage, days, wait):
cs = ClientStorage(addr, storage=storage, wait_for_server_on_startup=wait)
if wait:
# _startup() is an artifact of the way ZEO 1.0 works. The
# ClientStorage doesn't get fully initialized until registerDB()
# is called. The only thing we care about, though, is that
# registerDB() calls _startup().
cs._startup()
else:
connect(cs)
cs.invalidator = None
cs.pack(wait=1, days=days)
cs.close()
def _main(host, port, unix=None, days=1, username=None, password=None,
realm=None, blob_dir=None, storage='1', shared_blob_dir=True):
if unix is not None:
addr = unix
else:
if host is None:
host = socket.gethostname()
addr = host, int(port)
if blob_dir:
blob_dir = os.path.abspath(blob_dir)
wait = True
cs = None
try:
cs = ClientStorage(
addr, storage=storage, wait=wait, read_only=True,
username=username, password=password, realm=realm,
blob_dir=blob_dir, shared_blob_dir=shared_blob_dir,
)
cs.pack(wait=wait, days=int(days))
finally:
if cs is not None:
cs.close()
class DBMgr:
"""This class provides the access point to the Shelf (every client will
use this class in order to obtain a shelf) and some mechanism to
ensure there is only one connection opened to the DB during a single
request.
This class must not be instantiated, an instance can be obtained
though the "getInstance" method (implements the singleton pattern
to ensure unicity)
Needs to be checked if the class (static) attribute _instance is
thread-safe: as it is shared by all the objects of this class,
it could provoke concurrency troubles having 2 threads using the db
connection at the same time. However, the model under which we are
programming is not multi-threaded (mod_python seems to run different
interpreters for each apache subprocess, see mod_python doc section
4.1) so this thechnique can be used. This has to be taken into account
when migrating the system to a multi-threading environment.
"""
_instances = {}
def __init__(self, hostname=None, port=None, max_disconnect_poll=30):
# Please leave this import here, db.py is imported during installation process
from indico.core import config as Configuration
cfg = Configuration.Config.getInstance()
if not hostname:
hostname = cfg.getDBConnectionParams()[0]
if not port:
port = cfg.getDBConnectionParams()[1]
self._storage = ClientStorage((hostname, port), username=cfg.getDBUserName(),
password=cfg.getDBPassword(), realm=cfg.getDBRealm(),
max_disconnect_poll=max_disconnect_poll)
self._db = MigratedDB(self._storage)
self._conn = threading.local()
self._conn.conn = None
@classmethod
def getInstance(cls, *args, **kwargs):
pid = os.getpid()
if os.getpid() not in cls._instances:
from MaKaC.common.logger import Logger
Logger.get('dbmgr').debug('cls._instance is None')
cls._instances[pid] = DBMgr(*args, **kwargs)
return cls._instances[pid]
@classmethod
def setInstance(cls, dbInstance):
if dbInstance is None:
cls._instances.pop(os.getpid(), None)
else:
cls._instances[os.getpid()] = dbInstance
def _getConnObject(self):
return self._conn.conn
def _setConnObject(self, obj):
self._conn.conn = obj
def _delConnObject(self):
self._conn.conn = None
def startRequest(self):
"""Initialise the DB and starts a new transaction.
"""
self._conn.conn = self._db.open()
def endRequest(self, commit=True):
"""Closes the DB and commits changes.
"""
if commit:
self.commit()
else:
self.abort()
self._getConnObject().close()
self._delConnObject()
def getDBConnection(self):
return self._getConnObject()
def isConnected(self):
return hasattr(self._conn, 'conn') and self._conn.conn is not None
def getDBConnCache(self):
conn = self._getConnObject()
return conn._cache
def getDBClassFactory(self):
return self._db.classFactory
def commit(self, sub=False):
import StringIO
import transaction._transaction
transaction._transaction.StringIO = StringIO.StringIO
if (sub):
transaction.savepoint()
else:
transaction.commit()
def commitZODBOld(self, sub=False):
transaction.commit(sub)
def abort(self):
transaction.abort()
def sync(self):
self._getConnObject().sync()
def pack(self, days=1):
self._storage.pack(days=days)
def undoInfo(self, stepNumber=0):
# One step is made of 1000 transactions. First step is 0 and returns
# transactions 0 to 999.
return self._db.undoInfo(stepNumber*1000, (stepNumber+1)*1000)
def undo(self, trans_id):
self._db.undo(trans_id)
def getDBSize(self):
"""Return an approximate size of the database, in bytes."""
return self._storage.getSize()
def loadObject(self, oid, version):
return self._storage.load(oid, version)
def storeObject(self, oid, serial, data, version, trans):
return self._storage.store(oid, serial, data, version, trans)
def tpcBegin(self, trans):
self._storage.tpc_begin(trans)
def tpcVote(self, trans):
self._storage.tpc_vote(trans)
def tpcFinish(self, trans):
self._storage.tpc_finish(trans)
@contextmanager
def transaction(self, sync=False):
"""
context manager (`with`)
"""
if sync:
self.sync()
yield self.getDBConnection()
self.commit()
@contextmanager
def global_connection(self, commit=False):
"""Helper if you NEED a connection and don't know if one is available or not.
Useful e.g. in flask code that runs outside a request"""
if self.isConnected():
yield
else:
self.startRequest()
try:
yield
finally:
self.endRequest(commit)
# ZODB version check
try:
zodbPkg = pkg_resources.require('ZODB3')[0]
zodbVersion = zodbPkg.parsed_version
zodbVersion = (int(zodbVersion[0]), int(zodbVersion[1]))
except pkg_resources.DistributionNotFound:
# Very old versions, in which ZODB didn't register
# with pkg_resources
import ZODB
zodbVersion = ZODB.__version__.split('.')
if int(zodbVersion[0]) < 3:
raise Exception("ZODB 3 required! %s found" % zodbPkg.version)
elif int(zodbVersion[1]) < 7:
commit = commitZODBOld
class DBMgr:
"""This class provides the access point to the Shelf (every client will
use this class in order to obtain a shelf) and some mechanism to
ensure there is only one connection opened to the DB during a single
request.
This class must not be instantiated, an instance can be obtained
though the "getInstance" method (implements the singleton pattern
to ensure unicity)
Needs to be checked if the class (static) attribute _instance is
thread-safe: as it is shared by all the objects of this class,
it could provoke concurrency troubles having 2 threads using the db
connection at the same time. However, the model under which we are
programming is not multi-threaded (mod_python seems to run different
interpreters for each apache subprocess, see mod_python doc section
4.1) so this thechnique can be used. This has to be taken into account
when migrating the system to a multi-threading environment.
"""
_instances = {}
def __init__(self, hostname=None, port=None, max_disconnect_poll=30):
# Please leave this import here, db.py is imported during installation process
from indico.core.config import Config
cfg = Config.getInstance()
if not hostname:
hostname = cfg.getDBConnectionParams()[0]
if not port:
port = cfg.getDBConnectionParams()[1]
self._storage = ClientStorage((hostname, port),
username=cfg.getDBUserName(),
password=cfg.getDBPassword(),
realm=cfg.getDBRealm(),
max_disconnect_poll=max_disconnect_poll)
self._db = MigratedDB(self._storage)
self._conn = threading.local()
self._conn.conn = None
@classmethod
def getInstance(cls, *args, **kwargs):
pid = os.getpid()
if os.getpid() not in cls._instances:
cls._instances[pid] = DBMgr(*args, **kwargs)
return cls._instances[pid]
@classmethod
def setInstance(cls, dbInstance):
if dbInstance is None:
cls._instances.pop(os.getpid(), None)
else:
cls._instances[os.getpid()] = dbInstance
def _getConnObject(self):
return self._conn.conn
def _setConnObject(self, obj):
self._conn.conn = obj
def _delConnObject(self):
self._conn.conn = None
def startRequest(self):
"""Initialise the DB and starts a new transaction.
"""
self._conn.conn = self._db.open()
def endRequest(self, commit=True):
"""Closes the DB and commits changes.
"""
if commit:
self.commit()
else:
self.abort()
self._getConnObject().close()
self._delConnObject()
def getDBConnection(self):
return self._getConnObject()
def isConnected(self):
return hasattr(self._conn, 'conn') and self._conn.conn is not None
def getDBConnCache(self):
conn = self._getConnObject()
return conn._cache
def getDBClassFactory(self):
return self._db.classFactory
def commit(self, sub=False):
import StringIO
import transaction._transaction
transaction._transaction.StringIO = StringIO.StringIO
if (sub):
transaction.savepoint()
else:
transaction.commit()
def abort(self):
transaction.abort()
def sync(self):
self._getConnObject().sync()
def pack(self, days=1):
self._storage.pack(days=days)
def undoInfo(self, stepNumber=0):
# One step is made of 1000 transactions. First step is 0 and returns
# transactions 0 to 999.
return self._db.undoInfo(stepNumber * 1000, (stepNumber + 1) * 1000)
def undo(self, trans_id):
self._db.undo(trans_id)
def getDBSize(self):
"""Return an approximate size of the database, in bytes."""
return self._storage.getSize()
def loadObject(self, oid, version):
return self._storage.load(oid, version)
def storeObject(self, oid, serial, data, version, trans):
return self._storage.store(oid, serial, data, version, trans)
def tpcBegin(self, trans):
self._storage.tpc_begin(trans)
def tpcVote(self, trans):
self._storage.tpc_vote(trans)
def tpcFinish(self, trans):
self._storage.tpc_finish(trans)
@contextmanager
def transaction(self, sync=False):
"""
context manager (`with`)
"""
if sync:
self.sync()
yield self.getDBConnection()
self.commit()
@contextmanager
def global_connection(self, commit=False):
"""Helper if you NEED a connection and don't know if one is available or not.
Useful e.g. in flask code that runs outside a request"""
if self.isConnected():
yield
else:
self.startRequest()
try:
yield
finally:
self.endRequest(commit)
def pack2(addr, storage, days):
cs = ClientStorage(addr, storage=storage, wait=1, read_only=1)
cs.pack(wait=1, days=days)
cs.close()
def pack2(addr, storage, days):
cs = ClientStorage(addr, storage=storage, wait=1, read_only=1)
cs.pack(wait=1, days=days)
cs.close()
class DBMgr:
"""This class provides the access point to the Shelf (every client will
use this class in order to obtain a shelf) and some mechanism to
ensure there is only one connection opened to the DB during a single
request.
This class must not be instantiated, an instance can be obtained
though the "getInstance" method (implements the singleton pattern
to ensure unicity)
Needs to be checked if the class (static) attribute _instance is
thread-safe: as it is shared by all the objects of this class,
it could provoke concurrency troubles having 2 threads using the db
connection at the same time. However, the model under which we are
programming is not multi-threaded (mod_python seems to run different
interpreters for each apache subprocess, see mod_python doc section
4.1) so this thechnique can be used. This has to be taken into account
when migrating the system to a multi-threading environment.
"""
_instance = None
def __init__(self, hostname=None, port=None, max_disconnect_poll=30):
import Configuration # Please leave this import here, db.py is imported during installation process
cfg = Configuration.Config.getInstance()
if not hostname:
hostname = cfg.getDBConnectionParams()[0]
if not port:
port = cfg.getDBConnectionParams()[1]
self._storage = ClientStorage((hostname, port),
username=cfg.getDBUserName(),
password=cfg.getDBPassword(),
realm=cfg.getDBRealm(),
max_disconnect_poll=max_disconnect_poll)
self._db = MaKaCDB(self._storage)
self._conn = threading.local()
self._conn.conn = None
@classmethod
def getInstance(cls, *args, **kwargs):
if cls._instance == None:
Logger.get('dbmgr').debug('cls._instance is None')
cls._instance = DBMgr(*args, **kwargs)
return cls._instance
@classmethod
def setInstance(cls, dbInstance):
cls._instance = dbInstance
def _getConnObject(self):
return self._conn.conn
def _setConnObject(self, obj):
self._conn.conn = obj
def _delConnObject(self):
self._conn.conn = None
def startRequest(self):
"""Initialise the DB and starts a new transaction.
"""
self._conn.conn = self._db.open()
def endRequest(self, commit=True):
"""Closes the DB and commits changes.
"""
if commit:
self.commit()
else:
self.abort()
self._getConnObject().close()
self._delConnObject()
def getDBConnection(self):
return self._getConnObject()
def isConnected(self):
return hasattr(self._conn, 'conn') and self._conn.conn != None
def getDBConnCache(self):
conn = self._getConnObject()
return conn._cache
def getDBClassFactory(self):
return self._db.classFactory
def commit(self, sub=False):
if (sub):
transaction.savepoint()
else:
transaction.commit()
def commitZODBOld(self, sub=False):
transaction.commit(sub)
def abort(self):
transaction.abort()
def sync(self):
self._getConnObject().sync()
def pack(self, days=1):
self._storage.pack(days=days)
def undoInfo(self, stepNumber=0):
# One step is made of 1000 transactions. First step is 0 and returns
# transactions 0 to 999.
return self._db.undoInfo(stepNumber * 1000, (stepNumber + 1) * 1000)
def undo(self, trans_id):
self._db.undo(trans_id)
def getDBSize(self):
"""Return an approximate size of the database, in bytes."""
return self._storage.getSize()
def loadObject(self, oid, version):
return self._storage.load(oid, version)
def storeObject(self, oid, serial, data, version, trans):
return self._storage.store(oid, serial, data, version, trans)
def tpcBegin(self, trans):
self._storage.tpc_begin(trans)
def tpcVote(self, trans):
self._storage.tpc_vote(trans)
def tpcFinish(self, trans):
self._storage.tpc_finish(trans)
@contextmanager
def transaction(self, sync=False):
"""
context manager (`with`)
"""
if sync:
self.sync()
yield self.getDBConnection()
self.commit()
# ZODB version check
try:
zodbPkg = pkg_resources.require('ZODB3')[0]
zodbVersion = zodbPkg.parsed_version
zodbVersion = (int(zodbVersion[0]), int(zodbVersion[1]))
except pkg_resources.DistributionNotFound:
# Very old versions, in which ZODB didn't register
# with pkg_resources
import ZODB
zodbVersion = ZODB.__version__.split('.')
if int(zodbVersion[0]) < 3:
raise Exception("ZODB 3 required! %s found" % zodbPkg.version)
elif int(zodbVersion[1]) < 7:
commit = commitZODBOld
host = 'IP OR HOST'
port = PORT NUMBER
addr = host, int(port)
wait = True
days = 7
# Log initialize
logger = logging.getLogger('Instance Pack')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler(logpath)
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
logger.addHandler(fh)
cs = ClientStorage(addr,storage='1',wait=wait,read_only=True,username=None,password=None,realm=None)
logger.info('Pack initialize \n ---------------------------------')
cs.pack(wait=wait, days=int(days))
noSecurityManager()
transaction.savepoint(1)
transaction.commit()
app._p_jar.sync()
logger.info('Pack finally \n ---------------------------------')
class DBMgr:
"""This class provides the access point to the Shelf (every client will
use this class in order to obtain a shelf) and some mechanism to
ensure there is only one connection opened to the DB during a single
request.
This class must not be instantiated, an instance can be obtained
though the "getInstance" method (implements the singleton pattern
to ensure unicity)
Needs to be checked if the class (static) attribute _instance is
thread-safe: as it is shared by all the objects of this class,
it could provoke concurrency troubles having 2 threads using the db
connection at the same time. However, the model under which we are
programming is not multi-threaded (mod_python seems to run different
interpreters for each apache subprocess, see mod_python doc section
4.1) so this thechnique can be used. This has to be taken into account
when migrating the system to a multi-threading environment.
"""
_instance = None
def __init__( self, hostname=None, port=None ):
import Configuration # Please leave this import here, db.py is imported during installation process
cfg = Configuration.Config.getInstance()
if not hostname:
hostname = cfg.getDBConnectionParams()[0]
if not port:
port = cfg.getDBConnectionParams()[1]
self._storage=ClientStorage((hostname, port), username=cfg.getDBUserName(), password=cfg.getDBPassword(), realm=cfg.getDBRealm())
self._db=MaKaCDB(self._storage)
self._conn={}
@classmethod
def getInstance( cls, *args, **kwargs ):
if cls._instance == None:
Logger.get('dbmgr').debug('cls._instance is None')
cls._instance=DBMgr(*args, **kwargs)
return cls._instance
def _getConnObject(self):
tid=threading._get_ident()
if self._conn.has_key(tid):
return self._conn[tid]
return None
def _delConnObject(self):
tid=threading._get_ident()
del self._conn[tid]
def startRequest( self ):
"""Initialise the DB and starts a new transaction.
"""
conn = self._getConnObject()
if conn is None:
self._conn[threading._get_ident()]=self._db.open()
Logger.get('dbmgr').debug('Allocated connection for thread %s - table size is %s' % (threading._get_ident(), len(self._conn)))
else:
Logger.get('dbmgr').debug('Reused connection for thread %s - table size is %s' % (threading._get_ident(), len(self._conn)))
def endRequest( self, commit=True ):
"""Closes the DB and commits changes.
"""
if commit:
self.commit()
else:
self.abort()
#modification vendredi 010907
# try:
# self._conn.close()
# self._conn=None
# except:
# pass
self._getConnObject().close()
self._delConnObject()
def getDBConnection( self ):
conn = self._getConnObject()
if conn == None:
raise Exception( _("request not started"))
return conn
def isConnected( self ):
if self._getConnObject() == None:
return False
else:
return True
def getDBConnCache(self):
conn = self._getConnObject()
if conn == None:
raise Exception( _("request not started"))
return conn._cache
def getDBClassFactory(self):
return self._db.classFactory
def commit(self, sub=False):
if (sub):
transaction.savepoint()
else:
transaction.commit()
def commitZODBOld(self, sub=False):
transaction.commit(sub)
def abort(self):
transaction.abort()
def sync(self):
self._getConnObject().sync()
def pack( self, days=1 ):
self._storage.pack(days=days)
def undoInfo(self, stepNumber=0):
# One step is made of 1000 transactions. First step is 0 and returns
# transactions 0 to 999.
return self._db.undoInfo(stepNumber*1000, (stepNumber+1)*1000)
def undo(self, trans_id):
self._db.undo(trans_id)
def getDBSize( self ):
"""Return an approximate size of the database, in bytes."""
return self._storage.getSize()
def loadObject(self, oid, version):
return self._storage.load(oid, version)
def storeObject(self, oid, serial, data, version, trans):
return self._storage.store(oid, serial, data, version, trans)
def tpcBegin(self, trans):
self._storage.tpc_begin(trans)
def tpcVote(self, trans):
self._storage.tpc_vote(trans)
def tpcFinish(self, trans):
self._storage.tpc_finish(trans)
# ZODB version check
try:
zodbPkg = pkg_resources.require('ZODB3')[0]
zodbVersion = zodbPkg.parsed_version
zodbVersion = (int(zodbVersion[0]), int(zodbVersion[1]))
except pkg_resources.DistributionNotFound:
# Very old versions, in which ZODB didn't register
# with pkg_resources
import ZODB
zodbVersion = ZODB.__version__.split('.')
if int(zodbVersion[0]) < 3:
raise Exception("ZODB 3 required! %s found" % zodbPkg.version)
elif int(zodbVersion[1]) < 7:
commit = commitZODBOld
