def _check(args):
if not os.path.isdir('/proc'):
raise Exception('This command only works on systems that have /proc/')
with DBMgr.getInstance().global_connection():
status = Client().getStatus()
if status['hostname'] is not None and status[
'hostname'] != socket.getfqdn():
print >> sys.stderr, 'The daemon is running on another machine ({0[hostname]})'.format(
status)
sys.exit(2)
db_running = _check_running(False)
os_running = _check_running(True)
if not args.quiet:
print >> sys.stderr, 'Database status: running={1}, host={0[hostname]}, pid={0[pid]}'.format(
status, db_running)
print >> sys.stderr, 'Process status: running={0}'.format(
os_running)
if db_running and os_running:
print status['pid']
sys.exit(0)
elif not db_running and not os_running:
sys.exit(1)
elif db_running and not os_running:
if not args.quiet:
print >> sys.stderr, 'Marking dead scheduler as not running'
SchedulerModule.getDBInstance().setSchedulerRunningStatus(False)
DBMgr.getInstance().commit()
sys.exit(1)
else:
print >> sys.stderr, 'Unexpected state! Process is running, but scheduler is not marked as running'
sys.exit(2)
def category_cleanup():
cfg = Config.getInstance()
janitor_user = User.get_one(cfg.getJanitorUserId())
logger.debug("Checking whether any categories should be cleaned up")
for categ_id, days in cfg.getCategoryCleanup().iteritems():
try:
category = CategoryManager().getById(categ_id)
except KeyError:
logger.warning("Category %s does not exist!", categ_id)
continue
now = now_utc()
to_delete = [ev for ev in category.conferences if (now - ev._creationDS) > timedelta(days=days)]
if not to_delete:
continue
logger.info("Category %s: %s events were created more than %s days ago and will be deleted", categ_id,
len(to_delete), days)
for i, event in enumerate(to_delete, 1):
logger.info("Deleting %s", event)
event.delete(user=janitor_user)
if i % 100 == 0:
db.session.commit()
DBMgr.getInstance().commit()
db.session.commit()
DBMgr.getInstance().commit()
def _check(args):
if not os.path.isdir('/proc'):
raise Exception('This command only works on systems that have /proc/')
with DBMgr.getInstance().global_connection():
status = Client().getStatus()
if status['hostname'] is not None and status['hostname'] != socket.getfqdn():
print >>sys.stderr, 'The daemon is running on another machine ({0[hostname]})'.format(status)
sys.exit(2)
db_running = _check_running(False)
os_running = _check_running(True)
if not args.quiet:
print >>sys.stderr, 'Database status: running={1}, host={0[hostname]}, pid={0[pid]}'.format(
status, db_running)
print >>sys.stderr, 'Process status: running={0}'.format(os_running)
if db_running and os_running:
print status['pid']
sys.exit(0)
elif not db_running and not os_running:
sys.exit(1)
elif db_running and not os_running:
if not args.quiet:
print >>sys.stderr, 'Marking dead scheduler as not running'
SchedulerModule.getDBInstance().setSchedulerRunningStatus(False)
DBMgr.getInstance().commit()
sys.exit(1)
else:
print >>sys.stderr, 'Unexpected state! Process is running, but scheduler is not marked as running'
sys.exit(2)
def _process(self):
if not self._cancel:
if not self._confirmed:
p = RHMaintenancePack(self)
return p.display()
DBMgr.getInstance().pack()
self._redirect(urlHandlers.UHMaintenance.getURL())
def category_cleanup():
from indico.modules.events import Event
cfg = Config.getInstance()
janitor_user = User.get_one(cfg.getJanitorUserId())
logger.debug("Checking whether any categories should be cleaned up")
for categ_id, days in cfg.getCategoryCleanup().iteritems():
try:
category = Category.get(int(categ_id), is_deleted=False)
except KeyError:
logger.warning("Category %s does not exist!", categ_id)
continue
now = now_utc()
to_delete = Event.query.with_parent(category).filter(Event.created_dt < (now - timedelta(days=days))).all()
if not to_delete:
continue
logger.info("Category %s: %s events were created more than %s days ago and will be deleted", categ_id,
len(to_delete), days)
for i, event in enumerate(to_delete, 1):
logger.info("Deleting %s", event)
event.as_legacy.delete(user=janitor_user)
if i % 100 == 0:
db.session.commit()
DBMgr.getInstance().commit()
db.session.commit()
DBMgr.getInstance().commit()
def category_cleanup():
from indico.modules.events import Event
cfg = Config.getInstance()
janitor_user = User.get_one(cfg.getJanitorUserId())
logger.debug("Checking whether any categories should be cleaned up")
for categ_id, days in cfg.getCategoryCleanup().iteritems():
try:
category = Category.get(int(categ_id), is_deleted=False)
except KeyError:
logger.warning("Category %s does not exist!", categ_id)
continue
now = now_utc()
to_delete = Event.query.with_parent(category).filter(
Event.created_dt < (now - timedelta(days=days))).all()
if not to_delete:
continue
logger.info(
"Category %s: %s events were created more than %s days ago and will be deleted",
categ_id, len(to_delete), days)
for i, event in enumerate(to_delete, 1):
logger.info("Deleting %s", event)
event.as_legacy.delete(user=janitor_user)
if i % 100 == 0:
db.session.commit()
DBMgr.getInstance().commit()
db.session.commit()
DBMgr.getInstance().commit()
def rebuildRoomReservationsIndex():
from indico.core.db import DBMgr
from MaKaC.rb_location import CrossLocationDB
from MaKaC.rb_room import RoomBase
from MaKaC.plugins.RoomBooking.default.dalManager import DALManager
from BTrees.OOBTree import OOBTree
DBMgr.getInstance().startRequest()
CrossLocationDB.connect()
root = DALManager.getRoot()
resvEx = ReservationBase()
resvEx.isConfirmed = None
allResvs = CrossLocationQueries.getReservations(resvExample=resvEx)
print "There are " + str(len(allResvs)) + " resvs and pre-resvs to index..."
c = 0
root[_ROOM_RESERVATIONS_INDEX] = OOBTree()
print "Room => Reservations Index branch created"
for resv in allResvs:
roomReservationsIndexBTree = root[_ROOM_RESERVATIONS_INDEX]
resvs = roomReservationsIndexBTree.get(resv.room.id)
if resvs == None:
resvs = [] # New list of reservations for this room
roomReservationsIndexBTree.insert(resv.room.id, resvs)
resvs.append(resv)
roomReservationsIndexBTree[resv.room.id] = resvs
c += 1
if c % 100 == 0:
print c
CrossLocationDB.commit()
CrossLocationDB.disconnect()
DBMgr.getInstance().endRequest()
def category_cleanup():
cfg = Config.getInstance()
janitor_user = User.get_one(cfg.getJanitorUserId())
logger.debug("Checking whether any categories should be cleaned up")
for categ_id, days in cfg.getCategoryCleanup().iteritems():
try:
category = CategoryManager().getById(categ_id)
except KeyError:
logger.warning("Category {} does not exist!".format(categ_id))
continue
now = now_utc()
to_delete = [ev for ev in category.conferences if (now - ev._creationDS) > timedelta(days=days)]
if not to_delete:
continue
logger.info("Category {}: {} events were created more than {} days ago and will be deleted".format(
categ_id, len(to_delete), days
))
for i, event in enumerate(to_delete, 1):
logger.info("Deleting {}".format(event))
event.delete(user=janitor_user)
if i % 100 == 0:
db.session.commit()
DBMgr.getInstance().commit()
db.session.commit()
DBMgr.getInstance().commit()
def _process( self ):
if not self._cancel:
if not self._confirmed:
p=RHMaintenancePack(self)
return p.display()
DBMgr.getInstance().pack()
self._redirect(urlHandlers.UHMaintenance.getURL())
def _doValidate( self ):
#First, one must validate that the information is fine
errors = self._abstractData.check()
if errors:
p = abstracts.WPAbstractSubmission( self, self._target )
pars = self._abstractData.toDict()
pars["action"] = self._action
pars["attachments"] = []
for error in errors:
flash(error, 'error')
return p.display( **pars )
#Then, we create the abstract object and set its data to the one
# received
cfaMgr = self._target.getAbstractMgr()
abstract = cfaMgr.newAbstract( self._getUser() )
self._abstractData.setAbstractData(abstract)
#The commit must be forced before sending the confirmation
DBMgr.getInstance().commit()
#Email confirmation about the submission
mail.Mailer.send( _AbstractSubmissionNotification( abstract ), self._conf.getSupportInfo().getEmail(returnNoReply=True) )
#Email confirmation about the submission to coordinators
if cfaMgr.getSubmissionNotification().hasDestination():
asn=_AbstractSubmissionNotification( abstract )
asn.setSubject(_("[Indico] New abstract submission: %s")%asn.getDestination().getFullName())
mail.GenericMailer.send( asn )
#We must perform some actions: email warning to the authors
#Finally, we display a confirmation form
self._redirect( urlHandlers.UHAbstractSubmissionConfirmation.getURL( abstract ) )
def _doValidate(self):
#First, one must validate that the information is fine
errors = self._abstractData.check()
if errors:
p = abstracts.WPAbstractSubmission(self, self._target)
pars = self._abstractData.toDict()
pars["action"] = self._action
pars["attachments"] = []
for error in errors:
flash(error, 'error')
return p.display(**pars)
#Then, we create the abstract object and set its data to the one
# received
cfaMgr = self._target.getAbstractMgr()
abstract = cfaMgr.newAbstract(self._getUser())
self._abstractData.setAbstractData(abstract)
#The commit must be forced before sending the confirmation
DBMgr.getInstance().commit()
#Email confirmation about the submission
mail.Mailer.send(
_AbstractSubmissionNotification(abstract),
self._conf.getSupportInfo().getEmail(returnNoReply=True))
#Email confirmation about the submission to coordinators
if cfaMgr.getSubmissionNotification().hasDestination():
asn = _AbstractSubmissionNotification(abstract)
asn.setSubject(
_("[Indico] New abstract submission: %s") %
asn.getDestination().getFullName())
mail.GenericMailer.send(asn)
#We must perform some actions: email warning to the authors
#Finally, we display a confirmation form
self._redirect(
urlHandlers.UHAbstractSubmissionConfirmation.getURL(abstract))
def _invokeMethodRetryBefore(self):
# clear/init fossil cache
fossilize.clearCache()
# clear after-commit queue
flush_after_commit_queue(False)
# delete all queued emails
GenericMailer.flushQueue(False)
DBMgr.getInstance().sync()
def setUp(self):
DBMgr.getInstance().startRequest()
self.oldIndex = IndexesHolder()._getIdx()["categoryDateLtd"]
self.newIndex = IndexesHolder()._getIdx()["categoryDate"]
self.startDate = datetime(2010,5,13, 10, 0, 0, tzinfo=timezone('UTC'))
self.endDate = datetime(2010,5,14, 14, 0, 0, tzinfo=timezone('UTC'))
self.ch = ConferenceHolder()
self.categId = '0'
def _invokeMethodRetryBefore(self):
# clear/init fossil cache
fossilize.clearCache()
# clear after-commit queue
flush_after_commit_queue(False)
# delete all queued emails
GenericMailer.flushQueue(False)
DBMgr.getInstance().sync()
def remoteSynchronize(self, raiseExceptionOnSyncFail=False):
""" Calls the webcast synchronization URL
Will perform an intermediate commit before calling the URL
or the remote server will query a non-updated state of the indico DB.
raiseExceptionOnSyncFail: if True (default), we will raise a MaKaCError if calling the webcast
synchronization URL had a problem
"""
url = str(self.getWebcastSynchronizationURL()).strip()
if url:
try:
Logger.get('webcast').info("Doing an intermediate commit...")
DBMgr.getInstance().commit()
Logger.get('webcast').info("Commit done.")
Logger.get('webcast').info(
"Calling the webcast synchronization URL: " + url)
answer = urlopen(url, timeout=10).read(100000).strip()
Logger.get('webcast').info("Got answer: " + answer)
return answer
except HTTPError, e:
code = e.code
shortMessage = BaseHTTPRequestHandler.responses[code][0]
longMessage = BaseHTTPRequestHandler.responses[code][1]
Logger.get('webcast').error(
"""Calling the webcast synchronization URL: [%s] triggered HTTPError: %s (code = %s, shortMessage = '%s', longMessage = '%s'"""
% (str(url), str(e), code, shortMessage, longMessage))
if raiseExceptionOnSyncFail:
if str(code) == '404':
raise MaKaCError(
'Could not find the server at ' + str(url) +
"(HTTP error 404)", 'webcast')
elif str(code) == '500':
raise MaKaCError(
"The server at" + str(url) +
" has an internal problem (HTTP error 500)",
'webcast')
else:
raise MaKaCError(
"Problem contacting the webcast synchronization server. Reason: HTTPError: %s (code = %s, shortMessage = '%s', longMessage = '%s', url = '%s'"
"" % (str(e), code, shortMessage, longMessage,
str(url)), 'webcast')
except URLError, e:
Logger.get('webcast').error(
"""Calling the webcast synchronization URL: [%s] triggered exception: %s"""
% (str(url), str(e)))
if raiseExceptionOnSyncFail:
if str(e.reason).strip() == 'timed out':
raise MaKaCError(
"The webcast synchronization URL is not responding",
'webcast')
raise MaKaCError(
"""URLError when contacting the webcast synchronization URL: [%s]. Reason=[%s]"""
% (str(url), str(e.reason)), 'webcast')
def buildCategoryDateIndexLtd():
""" Builds limited version of CategoryDateIndex.
Can take a long time
"""
DBMgr.getInstance().startRequest()
idx = CategoryDateIndexLtd()
idx.buildIndex()
IndexesHolder()._getIdx()["categoryDateLtd"] = idx
DBMgr.getInstance().endRequest()
def buildCategoryDateIndexLtd():
""" Builds limited version of CategoryDateIndex.
Can take a long time
"""
DBMgr.getInstance().startRequest()
idx = CategoryDateIndexLtd()
idx.buildIndex()
IndexesHolder()._getIdx()["categoryDateLtd"] = idx
DBMgr.getInstance().endRequest()
def getConfIds():
DBMgr.getInstance().startRequest()
l = ConferenceHolder().getValuesToList()
ids = []
for conf in l:
ids.append(conf.getId())
DBMgr.getInstance().endRequest()
return ids
def register(interval=15):
from indico.modules.scheduler import Client
from dateutil.rrule import MINUTELY
from indico.core.db import DBMgr
DBMgr.getInstance().startRequest()
task = OutlookUpdateCalendarNotificationTask(MINUTELY, interval=interval)
client = Client()
client.enqueue(task)
DBMgr.getInstance().endRequest()
def remoteSynchronize(self, raiseExceptionOnSyncFail=False):
""" Calls the webcast synchronization URL
Will perform an intermediate commit before calling the URL
or the remote server will query a non-updated state of the indico DB.
raiseExceptionOnSyncFail: if True (default), we will raise a MaKaCError if calling the webcast
synchronization URL had a problem
"""
url = str(self.getWebcastSynchronizationURL()).strip()
if url:
try:
Logger.get("webcast").info("Doing an intermediate commit...")
DBMgr.getInstance().commit()
Logger.get("webcast").info("Commit done.")
Logger.get("webcast").info("Calling the webcast synchronization URL: " + url)
answer = urlopen(url, timeout=10).read(100000).strip()
Logger.get("webcast").info("Got answer: " + answer)
return answer
except HTTPError, e:
code = e.code
shortMessage = BaseHTTPRequestHandler.responses[code][0]
longMessage = BaseHTTPRequestHandler.responses[code][1]
Logger.get("webcast").error(
"""Calling the webcast synchronization URL: [%s] triggered HTTPError: %s (code = %s, shortMessage = '%s', longMessage = '%s'"""
% (str(url), str(e), code, shortMessage, longMessage)
)
if raiseExceptionOnSyncFail:
if str(code) == "404":
raise MaKaCError("Could not find the server at " + str(url) + "(HTTP error 404)", "webcast")
elif str(code) == "500":
raise MaKaCError(
"The server at" + str(url) + " has an internal problem (HTTP error 500)", "webcast"
)
else:
raise MaKaCError(
"Problem contacting the webcast synchronization server. Reason: HTTPError: %s (code = %s, shortMessage = '%s', longMessage = '%s', url = '%s'"
"" % (str(e), code, shortMessage, longMessage, str(url)),
"webcast",
)
except URLError, e:
Logger.get("webcast").error(
"""Calling the webcast synchronization URL: [%s] triggered exception: %s""" % (str(url), str(e))
)
if raiseExceptionOnSyncFail:
if str(e.reason).strip() == "timed out":
raise MaKaCError("The webcast synchronization URL is not responding", "webcast")
raise MaKaCError(
"""URLError when contacting the webcast synchronization URL: [%s]. Reason=[%s]"""
% (str(url), str(e.reason)),
"webcast",
)
def register(interval=15):
from indico.modules.scheduler import Client
from dateutil.rrule import MINUTELY
from indico.core.db import DBMgr
DBMgr.getInstance().startRequest()
task = OutlookUpdateCalendarNotificationTask(MINUTELY,
interval=interval)
client = Client()
client.enqueue(task)
DBMgr.getInstance().endRequest()
def _prepareDB(self):
# since the DBMgr instance will be replicated across objects,
# we just set it as None for this one.
# first, store the server address - this wouldn't normally be needed,
# but the tests won't work otherwise (as the DB is _not_ the default one)
hostname, port = DBMgr._instances[os.getppid()]._db.storage._addr
DBMgr.setInstance(DBMgr(hostname, port))
self._dbi = DBMgr.getInstance()
def _prepareDB(self):
# since the DBMgr instance will be replicated across objects,
# we just set it as None for this one.
# first, store the server address - this wouldn't normally be needed,
# but the tests won't work otherwise (as the DB is _not_ the default one)
hostname, port = DBMgr._instances[os.getppid()]._db.storage._addr
DBMgr.setInstance(DBMgr(hostname, port))
self._dbi = DBMgr.getInstance()
def invokeMethod(self, method, params):
cfg = Config.getInstance()
forced_conflicts = cfg.getForceConflicts()
max_retries = cfg.getMaxRetries()
result = None
self._invokeMethodBefore()
try:
for i, retry in enumerate(transaction.attempts(max_retries)):
with retry:
if i > 0:
# notify components that the request is being retried
signals.before_retry.send()
self._invokeMethodRetryBefore()
try:
try:
result = processRequest(method,
copy.deepcopy(params))
signals.after_process.send()
except NoReportError as e:
raise ServiceNoReportError(e.getMessage())
except (NoReportIndicoError, FormValuesError) as e:
raise ServiceNoReportError(e.getMessage(),
title=_("Error"))
# Raise a conflict error if enabled.
# This allows detecting conflict-related issues easily.
if i < forced_conflicts:
raise ConflictError
transaction.commit()
break
except ConflictError:
transaction.abort()
except ClientDisconnected:
transaction.abort()
time.sleep(i)
except DatabaseError:
handle_sqlalchemy_database_error()
break
self._invokeMethodSuccess()
except Exception as e:
transaction.abort()
if isinstance(e, CausedError):
raise
elif isinstance(e, ConstraintViolated):
raise ProcessError, ('ERR-P0', e.message), sys.exc_info()[2]
else:
raise ProcessError('ERR-P0', 'Error processing method.')
finally:
# notify components that the request has ended
DBMgr.getInstance().endRequest()
return result
def send_event_reminders():
reminders = EventReminder.find_all(~EventReminder.is_sent, EventReminder.scheduled_dt <= now_utc())
try:
for reminder in reminders:
logger.info('Sending event reminder: {}'.format(reminder))
reminder.send()
finally:
# If we fail at any point during the loop, we'll still commit
# the is_sent change for already-sent reminders instead of
# sending them over and over and thus spamming people.
db.session.commit()
DBMgr.getInstance().commit() # only needed for event email logs
def main( **kwargs ):
location = kwargs.get( 'location', 'Universe' )
from MaKaC.rb_factory import Factory
from indico.core.db import DBMgr
DBMgr.getInstance().startRequest()
Factory.getDALManager().connect()
initializeRoomBookingDB( location, force = True )
Factory.getDALManager().disconnect()
DBMgr.getInstance().endRequest()
def invokeMethod(self, method, params):
cfg = Config.getInstance()
forced_conflicts = cfg.getForceConflicts()
max_retries = cfg.getMaxRetries()
result = None
self._invokeMethodBefore()
try:
for i, retry in enumerate(transaction.attempts(max_retries)):
with retry:
if i > 0:
# notify components that the request is being retried
signals.before_retry.send()
self._invokeMethodRetryBefore()
try:
try:
result = processRequest(method, copy.deepcopy(params))
signals.after_process.send()
except NoReportError as e:
raise ServiceNoReportError(e.getMessage())
except (NoReportIndicoError, FormValuesError) as e:
raise ServiceNoReportError(e.getMessage(), title=_("Error"))
# Raise a conflict error if enabled.
# This allows detecting conflict-related issues easily.
if i < forced_conflicts:
raise ConflictError
transaction.commit()
break
except ConflictError:
transaction.abort()
except ClientDisconnected:
transaction.abort()
time.sleep(i)
except DatabaseError:
handle_sqlalchemy_database_error()
break
self._invokeMethodSuccess()
except Exception as e:
transaction.abort()
if isinstance(e, CausedError):
raise
elif isinstance(e, ConstraintViolated):
raise ProcessError, ('ERR-P0', e.message), sys.exc_info()[2]
else:
raise ProcessError('ERR-P0', 'Error processing method.')
finally:
# notify components that the request has ended
DBMgr.getInstance().endRequest()
return result
def main():
DBMgr.getInstance().startRequest()
ch = ConferenceHolder()
ids = []
print "Getting conference IDs..."
for conf in ch.getList():
ids.append(conf.getId())
totalNumberConfs=len(ids)
DBMgr.getInstance().endRequest()
print "Updating conferences..."
i = 0
N_CONF=10
cErrorList=[]
while ids:
if len(ids) >= N_CONF:
lids = ids[:N_CONF]
del ids[:N_CONF]
else:
lids = ids
ids = None
for j in range(10):
conf=None
try:
DBMgr.getInstance().startRequest()
for id in lids:
conf = ch.getById(id)
log("check for conference %s: %s/%s"%(conf.getId(),i,totalNumberConfs))
for cont in conf.getContributionList():
#if not isinstance(cont, AcceptedContribution):
if "content" in cont.getFields().keys():
if cont.getFields()["content"]:
if cont.getFields()["content"] != cont.description:
log(" contribution %s : content field no empty and diffrent from description"%cont.getId())
else:
#cont.setField("content",cont.description)
cont.getFields()["content"] = cont.description
cont._p_changed = 1
else:
#cont.setField("content",cont.description)
cont.getFields()["content"] = cont.description
cont._p_changed = 1
i += 1
DBMgr.getInstance().endRequest()
print "wait 0.5s..."
sleep(0.5)
break
except Exception, e:
cErrorList.append(conf)
i-=N_CONF
log("error %s, retry %d time(s)"%(e,int(10-j)))
sleep(int(j))
DBMgr.getInstance().abort()
def invokeMethod(self, method, params):
cfg = Config.getInstance()
forced_conflicts = cfg.getForceConflicts()
max_retries = cfg.getMaxRetries()
result = None
self._invokeMethodBefore()
try:
for i, retry in enumerate(transaction.attempts(max_retries)):
with retry:
if i > 0:
# notify components that the request is being retried
self._notify('requestRetry', i)
self._invokeMethodRetryBefore()
try:
# Raise a conflict error if enabled.
# This allows detecting conflict-related issues easily.
if i < forced_conflicts:
raise ConflictError
try:
result = processRequest(method,
copy.deepcopy(params))
except NoReportError as e:
raise ServiceNoReportError(e.getMsg())
except (NoReportIndicoError, FormValuesError) as e:
raise ServiceNoReportError(e.getMessage(),
title=_("Error"))
transaction.commit()
break
except ConflictError:
transaction.abort()
except ClientDisconnected:
transaction.abort()
time.sleep(i)
self._invokeMethodSuccess()
except Exception as e:
transaction.abort()
if isinstance(e, CausedError):
raise
else:
raise ProcessError('ERR-P0', 'Error processing method.')
finally:
# notify components that the request has ended
self._notify('requestFinished')
DBMgr.getInstance().endRequest()
return result
def tmp():
from MaKaC.rb_factory import Factory
from MaKaC.rb_room import RoomBase
from indico.core.db import DBMgr
from BTrees.OOBTree import OOBTree
DBMgr.getInstance().startRequest()
Factory.getDALManager().connect()
dayReservationsIndexBTree = OOBTree()
raise str(dir(dayReservationsIndexBTree))
Factory.getDALManager().disconnect()
DBMgr.getInstance().endRequest()
def setUp(self):
DBMgr.getInstance().startRequest()
self.oldIndex = IndexesHolder()._getIdx()["categoryDateLtd"]
self.newIndex = IndexesHolder()._getIdx()["categoryDate"]
self.startDate = datetime(2010,
5,
13,
10,
0,
0,
tzinfo=timezone('UTC'))
self.endDate = datetime(2010, 5, 14, 14, 0, 0, tzinfo=timezone('UTC'))
self.ch = ConferenceHolder()
self.categId = '0'
def buildCache(ids):
i = 1
for id in ids:
DBMgr.getInstance().startRequest()
try:
conf = ConferenceHolder().getById(id)
except:
print "conf %s not found"
continue
print i, ":", conf.getId()
og = outputGenerator(AccessWrapper())
x = og.confToXML(conf, 1, 1, 1, overrideCache=True)
y = og.confToXMLMarc21(conf, 1, 1, 1, overrideCache=True)
i += 1
DBMgr.getInstance().endRequest()
def buildCache(ids):
i = 1
for id in ids:
DBMgr.getInstance().startRequest()
try:
conf = ConferenceHolder().getById(id)
except:
print "conf %s not found"
continue
print i, ":", conf.getId()
og = outputGenerator(AccessWrapper())
x = og.confToXML(conf, 1, 1, 1, overrideCache=True)
y = og.confToXMLMarc21(conf, 1, 1, 1, overrideCache=True)
i += 1
DBMgr.getInstance().endRequest()
def main():
"""This script deletes existing category indexes and recreates them."""
DBMgr.getInstance().startRequest()
ch = CategoryManager()
for cat in ch.getList():
for conf in cat.getConferenceList():
chconf(conf)
get_transaction().commit()
# Tasks
# tl = Supervisor.
htl = timerExec.HelperTaskList.getTaskListInstance()
for task in htl.getTasks():
chtask(task)
DBMgr.getInstance().endRequest()
def invokeMethod(self, method, params):
cfg = Config.getInstance()
forced_conflicts = cfg.getForceConflicts()
max_retries = cfg.getMaxRetries()
result = None
self._invokeMethodBefore()
try:
for i, retry in enumerate(transaction.attempts(max_retries)):
with retry:
if i > 0:
# notify components that the request is being retried
self._notify('requestRetry', i)
self._invokeMethodRetryBefore()
try:
# Raise a conflict error if enabled.
# This allows detecting conflict-related issues easily.
if i < forced_conflicts:
raise ConflictError
try:
result = processRequest(method, copy.deepcopy(params))
except NoReportError as e:
raise ServiceNoReportError(e.getMsg())
except (NoReportIndicoError, FormValuesError) as e:
raise ServiceNoReportError(e.getMessage(), title=_("Error"))
transaction.commit()
break
except ConflictError:
transaction.abort()
except ClientDisconnected:
transaction.abort()
time.sleep(i)
self._invokeMethodSuccess()
except Exception as e:
transaction.abort()
if isinstance(e, CausedError):
raise
else:
raise ProcessError('ERR-P0', 'Error processing method.')
finally:
# notify components that the request has ended
self._notify('requestFinished')
DBMgr.getInstance().endRequest()
return result
def getInstance( self ):
dbroot = DBMgr.getInstance().getDBConnection().root()
if dbroot.has_key("adminlist"):
return dbroot["adminlist"]
al = _AdminList()
dbroot["adminlist"] = al
return al
def main(argv):
DBMgr.getInstance().startRequest()
print "Req start at " + str(datetime.now())
if "migrate" in argv:
migrateCategoryDateIndex()
if "switch" in argv:
switchIndex()
if "removeBackup" in argv:
deleteBackup()
if "display" in argv:
displayIndexes()
print "Req ends at " + str(datetime.now())
DBMgr.getInstance().endRequest()
def run(self):
sm = SyncManager.getDBInstance()
logger = self.getLogger()
# go over all the agents
for agtName, agent in sm.getAllAgents().iteritems():
# skip agents if they're not active
if not agent.isActive():
logger.warning("Agent '%s' is not active - skipping" % agtName)
continue
logger.info("Starting agent '%s'" % agtName)
try:
dbi = DBMgr.getInstance()
# pass the current time and a logger
result = agent.run(int_timestamp(nowutc()), logger=logger, dbi=dbi, task=self)
except:
logger.exception("Problem running agent '%s'" % agtName)
return
if result:
logger.info("Acknowledged successful operation")
agent.acknowledge()
dbi.commit()
else:
logger.info("'Acknowledge' not done - no records?")
logger.info("Agent '%s' finished" % agtName)
def _stop(args):
_setup(args)
running = _check_running()
if not args.force and not running:
raise Exception("The daemon doesn't seem to be running (consider -f?)")
dbi = DBMgr.getInstance()
dbi.startRequest()
c = Client()
c.shutdown(msg="Daemon script")
dbi.commit()
print "Waiting for death confirmation... "
for i in range(0, 20):
if not c.getStatus()['state']:
break
else:
time.sleep(1)
dbi.sync()
else:
print "FAILED!"
print "DONE!"
dbi.endRequest()
def remote_addr(self):
"""The remote address of the client."""
with DBMgr.getInstance().global_connection():
if HelperMaKaCInfo.getMaKaCInfoInstance().useProxy():
if self.access_route:
return self.access_route[0]
return super(IndicoRequest, self).remote_addr
def getVars(self):
vars = WTemplated.getVars(self)
vars["currentUser"] = self._currentuser
config = Config.getInstance()
imgLogin = config.getSystemIconURL("login")
vars["imgLogin"] = imgLogin
vars["isFrontPage"] = self._isFrontPage
vars["currentCategory"] = self.__currentCategory
vars['prot_obj'] = self._prot_obj
current_locale = get_current_locale()
vars["ActiveTimezone"] = session.timezone
"""
Get the timezone for displaying on top of the page.
1. If the user has "LOCAL" timezone then show the timezone
of the event/category. If that's not possible just show the
standard timezone.
2. If the user has a custom timezone display that one.
"""
vars["ActiveTimezoneDisplay"] = self._getTimezoneDisplay(
vars["ActiveTimezone"])
vars["SelectedLanguage"] = str(current_locale)
vars["SelectedLanguageName"] = current_locale.language_name
vars["Languages"] = get_all_locales()
if DBMgr.getInstance().isConnected():
vars["title"] = info.HelperMaKaCInfo.getMaKaCInfoInstance(
).getTitle()
vars["organization"] = info.HelperMaKaCInfo.getMaKaCInfoInstance(
).getOrganisation()
else:
vars["title"] = "Indico"
vars["organization"] = ""
vars['roomBooking'] = Config.getInstance().getIsRoomBookingActive()
vars['protectionDisclaimerProtected'] = legal_settings.get(
'network_protected_disclaimer')
vars['protectionDisclaimerRestricted'] = legal_settings.get(
'restricted_disclaimer')
#Build a list of items for the administration menu
adminItemList = []
if session.user and session.user.is_admin:
adminItemList.append({
'id': 'serverAdmin',
'url': urlHandlers.UHAdminArea.getURL(),
'text': _("Server admin")
})
vars["adminItemList"] = adminItemList
vars['extra_items'] = HeaderMenuEntry.group(
values_from_signal(signals.indico_menu.send()))
vars["getProtection"] = self._getProtection
vars["show_contact"] = config.getPublicSupportEmail() is not None
return vars
def get_context(self):
if self._context is None:
self._context = context = {}
self._info = []
add_to_context = partial(_add_to_context, context, self._info)
for attr in ('date', 'time', 'datetime', 'timedelta'):
add_to_context(getattr(datetime, attr), doc='stdlib datetime.{}'.format(attr), color='yellow')
add_to_context(Conference)
add_to_context(ConferenceHolder)
add_to_context(CategoryManager)
add_to_context(AvatarHolder)
add_to_context(GroupHolder)
add_to_context(Catalog)
add_to_context(IndexesHolder)
add_to_context(IndicoConfigWrapper(Config.getInstance()), 'config')
add_to_context(LocalProxy(HelperMaKaCInfo.getMaKaCInfoInstance), 'minfo')
add_to_context(current_app, 'app')
for name, cls in sorted(db.Model._decl_class_registry.iteritems(), key=itemgetter(0)):
if hasattr(cls, '__table__'):
add_to_context(cls, color='cyan')
add_to_context(DBMgr.getInstance(), 'dbi', doc='zodb db interface', color='cyan!')
add_to_context(db, 'db', doc='sqlalchemy db interface', color='cyan!')
add_to_context(transaction, doc='transaction module', color='cyan!')
signals.plugin.shell_context.send(add_to_context=add_to_context)
return self._context
def _event_or_shorturl(confId, shorturl_namespace=False, ovw=False):
from MaKaC.conference import ConferenceHolder
from MaKaC.common.url import ShortURLMapper
with DBMgr.getInstance().global_connection():
ch = ConferenceHolder()
su = ShortURLMapper()
if ch.hasKey(confId):
# For obvious reasons an event id always comes first.
# If it's used within the short url namespace we redirect to the event namespace, otherwise
# we call the RH to display the event
if shorturl_namespace:
url = UHConferenceDisplay.getURL(ch.getById(confId))
func = lambda: redirect(url)
else:
params = request.args.to_dict()
params['confId'] = confId
if ovw:
params['ovw'] = 'True'
func = lambda: conferenceDisplay.RHConferenceDisplay(None).process(params)
elif (shorturl_namespace or app.config['INDICO_COMPAT_ROUTES']) and su.hasKey(confId):
if shorturl_namespace:
# Correct namespace => redirect to the event
url = UHConferenceDisplay.getURL(su.getById(confId))
func = lambda: redirect(url)
else:
# Old event namespace => 301-redirect to the new shorturl first to get Google etc. to update it
url = url_for('.shorturl', confId=confId)
func = lambda: redirect(url, 301)
else:
raise NotFound(
_('The specified event with id or tag "%s" does not exist or has been deleted') % confId)
return func()
def timezone(self):
if '_timezone' in self:
return self['_timezone']
if '_avatarId' not in self:
return 'LOCAL'
with DBMgr.getInstance().global_connection():
return HelperMaKaCInfo.getMaKaCInfoInstance().getTimezone()
def cacheDay(dest, day):
"""
Cache a day, by calling wget in "mirror" mode
"""
dbi = DBMgr.getInstance()
dbi.startRequest()
index = {}
calIdx = IndexesHolder().getIndex('calendar')
objs = calIdx.getObjectsInDay(day)
for confId in objs:
if confId == '': continue
obj = ConferenceHolder().getById(confId)
url = str(urlHandlers.UHConferenceDisplay.getURL(obj))
savedDirs = re.match(r'http:\/\/(.*)', url).group(1).split('/')
print "Calling wget for %s..." % url
os.system(WGET_COMMAND % (confId, url,
os.path.join(dest, confId),
savedDirs[0]))
print "done!"
index[confId] = (os.path.join(confId,*savedDirs)+'.html', obj.getTitle())
dbi.endRequest(False)
return index
def _show(args):
dbi = DBMgr.getInstance()
dbi.startRequest()
c = Client()
if args.field == "status":
status = c.getStatus()
if status['state']:
print 'Scheduler is currently running on {0[hostname]} with pid {0[pid]}'.format(status)
else:
print 'Scheduler is currently NOT running'
print """
Spooled commands: %(spooled)s
Tasks:
- Waiting: %(waiting)s
- Running: %(running)s
- Failed: %(failed)s
- Finished: %(finished)s
""" % status
elif args.field == "spool":
for op, obj in c.getSpool():
if op in ['add', 'del']:
print "%s %s" % (op, obj)
else:
print op
dbi.endRequest()
def _show(args):
dbi = DBMgr.getInstance()
dbi.startRequest()
c = Client()
if args.field == "status":
status = c.getStatus()
if status['state']:
print 'Scheduler is currently running on {0[hostname]} with pid {0[pid]}'.format(
status)
else:
print 'Scheduler is currently NOT running'
print """
Spooled commands: %(spooled)s
Tasks:
- Waiting: %(waiting)s
- Running: %(running)s
- Failed: %(failed)s
- Finished: %(finished)s
""" % status
elif args.field == "spool":
for op, obj in c.getSpool():
if op in ['add', 'del']:
print "%s %s" % (op, obj)
else:
print op
dbi.endRequest()
def main():
dbi = DBMgr.getInstance()
dbi.startRequest()
ch = ConferenceHolder()
totalSize = 0
fNumber = 0
for __, obj in conferenceHolderIterator(ch, verbose=True):
for material in obj.getAllMaterialList():
for res in material.getResourceList():
if isinstance(res, LocalFile):
try:
totalSize += res.getSize()
fNumber += 1
except OSError:
print "Problems stating size of '%s'" % res.getFilePath(
)
dbi.endRequest(False)
print "%d files, %d bytes total" % (fNumber, totalSize)
print "avg %s bytes/file" % (float(totalSize) / fNumber)
def set_best_lang():
"""
Get the best language/locale for the current user. This means that first
the session will be checked, and then in the absence of an explicitly-set
language, we will try to guess it from the browser settings and only
after that fall back to the server's default.
"""
if not has_request_context():
return 'en_GB' if current_app.config[
'TESTING'] else HelperMaKaCInfo.getMaKaCInfoInstance().getLang()
elif session.lang is not None:
return session.lang
# try to use browser language
preferred = [
x.replace('-', '_') for x in request.accept_languages.values()
]
resolved_lang = negotiate_locale(preferred,
list(get_all_locales()),
aliases=LOCALE_ALIASES)
if not resolved_lang:
if current_app.config['TESTING']:
return 'en_GB'
with DBMgr.getInstance().global_connection():
# fall back to server default
minfo = HelperMaKaCInfo.getMaKaCInfoInstance()
resolved_lang = minfo.getLang()
session.lang = resolved_lang
return resolved_lang
def remote_addr(self):
"""The remote address of the client."""
with DBMgr.getInstance().global_connection():
if HelperMaKaCInfo.getMaKaCInfoInstance().useProxy():
if self.access_route:
return self.access_route[0]
return super(IndicoRequest, self).remote_addr
def getToList():
toList = []
DBMgr.getInstance().startRequest()
ch = conference.ConferenceHolder()
c = ch.getById("0")
toList = []
for contrib in c.getContributionList():
if not isinstance(contrib.getCurrentStatus(), conference.ContribStatusWithdrawn):
for pa in contrib.getPrimaryAuthorList():
if pa.getEmail().strip() != "" and (not pa.getEmail() in toList):
toList.append(pa.getEmail())
for spk in contrib.getSpeakerList():
if spk.getEmail().strip() != "" and (not spk.getEmail() in toList):
toList.append(spk.getEmail())
DBMgr.getInstance().endRequest()
return toList
def run(agent_id, force=False):
"""Runs the livesync agent"""
update_session_options(db)
if agent_id is None:
agent_list = LiveSyncAgent.find_all()
else:
agent = LiveSyncAgent.find_first(id=int(agent_id))
if agent is None:
print 'No such agent'
return
agent_list = [agent]
for agent in agent_list:
if agent.backend is None:
print cformat(
'Skipping agent: %{red!}{}%{reset} (backend not found)'
).format(agent.name)
continue
if not agent.initial_data_exported and not force:
print cformat(
'Skipping agent: %{red!}{}%{reset} (initial export not performed)'
).format(agent.name)
continue
print cformat('Running agent: %{white!}{}%{reset}').format(agent.name)
with DBMgr.getInstance().global_connection():
try:
agent.create_backend().run()
db.session.commit()
finally:
transaction.abort()
def set_best_lang():
"""
Get the best language/locale for the current user. This means that first
the session will be checked, and then in the absence of an explicitly-set
language, we will try to guess it from the browser settings and only
after that fall back to the server's default.
"""
if not has_request_context():
return 'en_GB' if current_app.config['TESTING'] else HelperMaKaCInfo.getMaKaCInfoInstance().getLang()
elif session.lang is not None:
return session.lang
# try to use browser language
preferred = [x.replace('-', '_') for x in request.accept_languages.values()]
resolved_lang = negotiate_locale(preferred, list(get_all_locales()), aliases=LOCALE_ALIASES)
if not resolved_lang:
if current_app.config['TESTING']:
return 'en_GB'
with DBMgr.getInstance().global_connection():
# fall back to server default
minfo = HelperMaKaCInfo.getMaKaCInfoInstance()
resolved_lang = minfo.getLang()
session.lang = resolved_lang
return resolved_lang
def getInstance(self):
dbroot = DBMgr.getInstance().getDBConnection().root()
if dbroot.has_key("adminlist"):
return dbroot["adminlist"]
al = _AdminList()
dbroot["adminlist"] = al
return al
def _run(args):
_setup(args)
formatter = logging.Formatter("%(asctime)s %(name)s - %(levelname)s %(filename)s:%(lineno)s: %(message)s")
root = logging.getLogger('')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
root.addHandler(handler)
dbi = DBMgr.getInstance(max_disconnect_poll=40)
dbi.startRequest()
info = HelperMaKaCInfo.getMaKaCInfoInstance()
useRBDB = info.getRoomBookingModuleActive()
if useRBDB:
DALManager.connect()
sm = SchedulerModule.getDBInstance()
t = sm.getTaskById(args.taskid)
t.plugLogger(logging.getLogger('console.run/%s' % args.taskid))
t.run()
if useRBDB:
DALManager.commit()
DALManager.disconnect()
dbi.endRequest()
def _stop(args):
_setup(args)
running = _check_running()
if not args.force and not running:
raise Exception("The daemon doesn't seem to be running (consider -f?)")
dbi = DBMgr.getInstance()
dbi.startRequest()
c = Client()
c.shutdown(msg="Daemon script")
dbi.commit()
print "Waiting for death confirmation... "
for i in range(0, 20):
if not c.getStatus()['state']:
break
else:
time.sleep(1)
dbi.sync()
else:
print "FAILED!"
print "DONE!"
dbi.endRequest()
def main(argv):
DBMgr.getInstance().startRequest()
print "Req start at " + str(datetime.now())
if "migrate" in argv:
migrateCategoryDateIndex()
if "switch" in argv:
switchIndex()
if "removeBackup" in argv:
deleteBackup()
if "display" in argv:
displayIndexes()
print "Req ends at " + str(datetime.now())
DBMgr.getInstance().endRequest()
def cacheDay(dest, day):
"""
Cache a day, by calling wget in "mirror" mode
"""
dbi = DBMgr.getInstance()
dbi.startRequest()
index = {}
calIdx = IndexesHolder().getIndex('calendar')
objs = calIdx.getObjectsInDay(day)
for confId in objs:
if confId == '': continue
obj = ConferenceHolder().getById(confId)
url = str(urlHandlers.UHConferenceDisplay.getURL(obj))
savedDirs = re.match(r'http:\/\/(.*)', url).group(1).split('/')
print "Calling wget for %s..." % url
os.system(WGET_COMMAND % (confId, url,
os.path.join(dest, confId),
savedDirs[0]))
print "done!"
index[confId] = (os.path.join(confId,*savedDirs)+'.html', obj.getTitle())
dbi.endRequest(False)
return index
def getInstance(self):
dbroot = DBMgr.getInstance().getDBConnection().root()
if dbroot.has_key("ecosoclist"):
return dbroot["ecosoclist"]
al = _ECOSOCList()
dbroot["ecosoclist"] = al
return al
