def index_site(request, autocommit=True):
"""Index all site contents in internal catalog"""
application = site_factory(request)
if application is not None:
try:
set_local_registry(application.getSiteManager())
catalog = get_utility(ICatalog)
catalog.reset()
transaction.savepoint()
intids = get_utility(IIntIds)
for index, document in enumerate(find_objects_providing(application, Interface)):
if INoAutoIndex.providedBy(document):
continue
if IBroken.providedBy(document):
print("Skipping broken object: {0!r}".format(document))
else:
print("Indexing: {0!r}".format(document))
catalog.reindex_doc(intids.register(document), document)
if not index % 100:
transaction.savepoint()
finally:
set_local_registry(None)
if autocommit:
transaction.commit()
return application
def evolve(site):
"""Evolve 1: remove all images thumbnails to free blobs"""
registry = get_local_registry()
try:
medias = set()
set_local_registry(site.getSiteManager())
LOGGER.warning(
"Removing all thumbnails from database to free unused blobs...")
intids = get_utility(IIntIds)
for ref in list(intids.refs.keys()):
obj = intids.queryObject(ref)
if IMediaFile.providedBy(obj):
LOGGER.debug(
">>> removing thumbnails for image {!r}".format(obj))
thumbnails = IThumbnails(obj, None)
if thumbnails is not None:
medias.add(obj)
thumbnails.clear_thumbnails()
LOGGER.warning(
"Thumbnails cleanup is finished. Launch *zeopack* (for ZEO storage) "
"or *zodbpack* (for Relstorage) command to remove all unused blobs."
)
LOGGER.warning("{} images updated".format(len(medias)))
finally:
set_local_registry(registry)
def site_factory(request):
"""Application site factory
On application startup, this factory checks configuration to get application name and
load it from the ZODB; if the application can't be found, configuration is scanned to
get application factory, create a new one and create a local site manager.
"""
conn = get_connection(request)
root = conn.root()
application_key = request.registry.settings.get(PYAMS_APPLICATION_SETTINGS_KEY,
PYAMS_APPLICATION_DEFAULT_NAME)
application = root.get(application_key)
if application is None:
factory = request.registry.settings.get(PYAMS_APPLICATION_FACTORY_KEY)
if factory:
resolver = DottedNameResolver()
factory = resolver.maybe_resolve(factory)
else:
factory = request.registry.queryUtility(ISiteRootFactory, default=BaseSiteRoot)
application = root[application_key] = factory()
if IPossibleSite.providedBy(application):
lsm = LocalSiteManager(application, default_folder=False)
application.setSiteManager(lsm)
try:
# if some components require a valid and complete registry
# with all registered utilities, they can subscribe to
# INewLocalSiteCreatedEvent event interface
set_local_registry(application.getSiteManager())
get_current_registry().notify(NewLocalSiteCreatedEvent(application))
finally:
set_local_registry(None)
import transaction # pylint: disable=import-outside-toplevel
transaction.commit()
return application
def evolve(site):
"""Evolve 1: update roles annotations"""
registry = get_local_registry()
try:
set_local_registry(site.getSiteManager())
for content in find_objects_providing(site, IDefaultProtectionPolicy):
annotations = IAnnotations(content)
if ROLES_ANNOTATIONS_KEY in annotations:
LOGGER.info("Updating roles annotations for {!r}".format(content))
annotations[POLICY_ANNOTATIONS_KEY] = annotations[ROLES_ANNOTATIONS_KEY]
del annotations[ROLES_ANNOTATIONS_KEY]
finally:
set_local_registry(registry)
def evolve(site):
"""Evolve 2: create reference for all files blobs"""
registry = get_local_registry()
try:
files = set()
set_local_registry(site.getSiteManager())
LOGGER.warning("Creating references to all blobs...")
intids = get_utility(IIntIds)
references = get_utility(IBlobReferenceManager)
for ref in list(intids.refs.keys()):
obj = intids.queryObject(ref)
if IFile.providedBy(obj):
blob = getattr(obj, '_blob', None)
if blob is not None:
references.add_reference(blob, obj)
LOGGER.debug(">>> updated blob reference for file {!r}".format(obj))
LOGGER.warning("{} files updated".format(len(files)))
finally:
set_local_registry(registry)
def upgrade_site(request):
"""Upgrade site when needed
This function is executed by *pyams_upgrade* console script.
Site generations are registered named utilities providing
:py:class:`ISiteGenerations <pyams_utils.interfaces.site.ISiteGenerations>` interface.
Current site generations are stored into annotations for each generation adapter.
"""
application = site_factory(request)
if application is not None:
try:
set_local_registry(application.getSiteManager())
generations = get_annotation_adapter(application,
SITE_GENERATIONS_KEY,
PersistentMapping,
notify=False,
locate=False)
for name, utility in sorted(get_utilities_for(ISiteGenerations),
key=lambda x: x[1].order):
if not name:
name = '.'.join(
(utility.__module__, utility.__class__.__name__))
current = generations.get(name)
if not current:
print("Upgrading {0} to generation {1}...".format(
name, utility.generation))
elif current < utility.generation:
print("Upgrading {0} from generation {1} to {2}...".format(
name, current, utility.generation))
utility.evolve(application, current)
generations[name] = utility.generation
finally:
set_local_registry(None)
import transaction # pylint: disable=import-outside-toplevel
transaction.commit()
return application
def _run(self, report, **kwargs): # pylint: disable=too-many-locals
"""Task execution wrapper"""
status = TASK_STATUS_NONE
result = None
# initialize ZCA hook
registry = kwargs.pop('registry')
manager.push({'registry': registry, 'request': None})
config = Configurator(registry=registry)
config.hook_zca()
# open ZODB connection
zodb_connection = ZODBConnection(name=kwargs.pop('zodb_name', ''))
with zodb_connection as root:
try:
application_name = registry.settings.get(PYAMS_APPLICATION_SETTINGS_KEY,
PYAMS_APPLICATION_DEFAULT_NAME)
application = root.get(application_name)
sm = application.getSiteManager() # pylint: disable=invalid-name
scheduler_util = sm.get(SCHEDULER_NAME)
task = scheduler_util.get(self.__name__)
if task is not None:
set_local_registry(sm)
request = check_request(base_url=scheduler_util.notified_host,
registry=registry, principal_id=self.principal_id)
request.root = application
with RequestContext(request):
if not (kwargs.get('run_immediate') or task.is_runnable()):
LOGGER.debug("Skipping inactive task {0}".format(task.name))
return status, result
translate = request.localizer.translate
tm = ITransactionManager(task) # pylint: disable=invalid-name
for attempt in tm.attempts():
with attempt as t: # pylint: disable=invalid-name
start_date = datetime.utcnow()
duration = 0.
try:
registry.notify(BeforeRunJobEvent(task))
(status, result) = task.run(report, **kwargs)
end_date = datetime.utcnow()
duration = (end_date - start_date).total_seconds()
report.write('\n\nTask duration: {0}'.format(
get_duration(start_date, request=request)))
if scheduler_util.notified_host and (ChatMessage is not None):
message = ChatMessage(
request=request,
host=scheduler_util.notified_host,
action='notify',
category='scheduler.run',
status='success',
source=INTERNAL_USER_ID,
title=translate(_("Task execution")),
message=translate(_("Task '{}' was executed without "
"error")).format(task.name),
url='/'.join(('', '++etc++site',
scheduler_util.__name__, 'admin'))
)
message.send()
except: # pylint: disable=bare-except
status = TASK_STATUS_ERROR
# pylint: disable=protected-access
task._log_exception(report,
"An error occurred during execution of "
"task '{0}'".format(task.name))
if scheduler_util.notified_host and (ChatMessage is not None):
message = ChatMessage(
request=request,
host=scheduler_util.notified_host,
action='notify',
category='scheduler.run',
status='danger',
source=INTERNAL_USER_ID,
title=translate(_("Task execution")),
message=translate(_("An error occurred during "
"execution of task '{}'"
"")).format(task.name),
url='/'.join(('', '++etc++site',
scheduler_util.__name__, 'admin'))
)
message.send()
registry.notify(AfterRunJobEvent(task, status, result))
task.store_report(report, status, start_date, duration)
task.send_report(report, status, registry)
if t.status == 'Committed':
break
except: # pylint: disable=bare-except
self._log_exception(None, "Can't execute scheduled job {0}".format(self.name))
tm = ITransactionManager(self, None) # pylint: disable=invalid-name
if tm is not None:
tm.abort()
return status, result
def handle_new_application(event): # pylint: disable=unused-argument,too-many-locals,too-many-branches
"""Start scheduler process when application is created"""
# Check for PyAMS command line script
cmdline = os.path.split(sys.argv[0])[-1]
if cmdline.startswith('pyams_'):
return
registry = get_pyramid_registry()
settings = registry.settings
start_handler = asbool(settings.get(SCHEDULER_STARTER_KEY, False))
if not start_handler:
return
# check if port is available
handler_address = settings.get(SCHEDULER_HANDLER_KEY, '127.0.0.1:5555')
hostname, port = handler_address.split(':')
if is_port_in_use(int(port), hostname):
LOGGER.warning("Scheduler port already used, aborting...")
return
# get database connection
connection = get_connection_from_settings(settings)
root = connection.root()
# get application
application_name = settings.get(PYAMS_APPLICATION_SETTINGS_KEY,
PYAMS_APPLICATION_DEFAULT_NAME)
application = root.get(application_name)
if application is None:
return
process = None
sm = application.getSiteManager() # pylint: disable=invalid-name
set_local_registry(sm)
try:
scheduler_util = sm.get(SCHEDULER_NAME)
if scheduler_util is None:
return
try:
zodb_name = scheduler_util.zodb_name
except ComponentLookupError:
pass
else:
# create scheduler process
process = SchedulerProcess(handler_address,
SchedulerMessageHandler,
settings.get(SCHEDULER_AUTH_KEY),
settings.get(SCHEDULER_CLIENTS_KEY),
registry)
# load tasks
for task in scheduler_util.values():
if not task.is_runnable():
continue
trigger = task.get_trigger()
LOGGER.debug(
"Adding scheduler job for task '{0.name}'".format(task))
process.scheduler.add_job(task,
trigger,
id=str(task.internal_id),
name=task.name,
kwargs={
'zodb_name': zodb_name,
'registry': registry
})
# start process
LOGGER.info("Starting tasks scheduler {0!r}...".format(process))
process.start()
if process.is_alive():
atexit.register(process_exit_func, process=process)
LOGGER.info("Started tasks scheduler with PID {0}.".format(
process.pid))
finally:
if process and not process.is_alive():
process.terminate()
process.join()
set_local_registry(None)