async def initialize_catalog(self, site):
await self.remove_catalog(site)
mappings = get_mappings()
index_name = self.get_index_name(site)
version = self.get_version(site)
real_index_name = index_name + '_' + str(version)
index_settings = DEFAULT_SETTINGS.copy()
index_settings.update(app_settings.get('index', {}))
try:
await self.conn.indices.create(real_index_name)
except TransportError as e:
logger.warn('Transport Error', exc_info=e)
except ConnectionError:
logger.warn('elasticsearch not installed', exc_info=True)
pass
except RequestError:
pass
try:
await self.conn.indices.put_alias(index_name, real_index_name)
except TransportError as e:
logger.warn('Transport Error', exc_info=e)
except ConnectionError:
logger.warn('elasticsearch not installed', exc_info=True)
pass
except RequestError:
pass
try:
await self.conn.indices.close(index_name)
except TransportError as e:
logger.warn('Transport Error', exc_info=e)
except ConnectionError:
logger.warn('elasticsearch not installed', exc_info=True)
pass
except RequestError:
pass
try:
await self.conn.indices.put_settings(index_settings, index_name)
for key, value in mappings.items():
await self.conn.indices.put_mapping(index_name, key, value)
except TransportError as e:
logger.warn('Transport Error', exc_info=e)
except ConnectionError:
logger.warn('elasticsearch not installed', exc_info=True)
return
except RequestError:
return
await self.conn.indices.open(index_name)
await self.conn.cluster.health(wait_for_status='yellow')
self.set_index_name(site, index_name)
def get_mappings():
from plone.server import app_settings
mapping_overrides = app_settings.get('elasticsearch',
{}).get('mapping_overrides', {})
# Mapping calculated from schemas
global_mappings = {}
base_type_overrides = mapping_overrides.get('*', {})
for name, schema in getUtilitiesFor(IResourceFactory):
# For each type
mappings = {}
type_overrides = base_type_overrides.copy()
type_overrides.update(mapping_overrides.get(name, {}))
for field_name, catalog_info in get_index_fields(name).items():
catalog_type = catalog_info.get('type', 'text')
field_mapping = catalog_info.get('field_mapping', None)
if field_mapping is None:
field_mapping = CATALOG_TYPES[catalog_type]
if field_name in type_overrides:
field_mapping = type_overrides[field_name]
mappings[field_name] = field_mapping
global_mappings[name] = {'properties': mappings}
return global_mappings
async def migrate_index(self, site, sndstep=False):
index_name = self.get_index_name(site)
version = self.get_version(site)
mappings = get_mappings()
index_settings = DEFAULT_SETTINGS.copy()
index_settings.update(app_settings.get('index', {}))
next_version = version + 1
real_index_name = index_name + '_' + str(version)
real_index_name_next_version = index_name + '_' + str(next_version)
temp_index = index_name + '_' + str(next_version) + '_t'
# Create and setup the new index
# exists = await self.conn.indices.exists(index_name)
# if exists:
# logger.warn('Canonical index exist')
# await self.conn.indices.delete(index_name)
# Create and setup the new index
if sndstep is False:
exists = await self.conn.indices.exists(real_index_name_next_version)
if exists:
logger.warn('New version exist')
await self.conn.indices.delete(real_index_name_next_version)
exists = await self.conn.indices.exists(temp_index)
conn_es = await self.conn.transport.get_connection()
if exists:
# There is a temp index so it needs to be reindex to the old one
# Its been a failing reindexing
logger.warn('Temp index exist')
# Move aliases
body = {
"actions": [
{"remove": {
"alias": index_name,
"index": temp_index
}},
{"add": {
"alias": index_name,
"index": real_index_name
}}
]
}
conn_es = await self.conn.transport.get_connection()
async with conn_es._session.post(
conn_es._base_url + '_aliases',
data=json.dumps(body),
timeout=1000000
) as resp:
pass
body = {
"source": {
"index": temp_index
},
"dest": {
"index": real_index_name
}
}
async with conn_es._session.post(
conn_es._base_url + '_reindex',
data=json.dumps(body)
) as resp:
pass
await self.conn.indices.delete(temp_index)
await self.conn.indices.create(temp_index)
await self.conn.indices.create(real_index_name_next_version)
await self.conn.indices.close(real_index_name_next_version)
await self.conn.indices.put_settings(
index_settings, real_index_name_next_version)
for key, value in mappings.items():
await self.conn.indices.put_mapping(
real_index_name_next_version, key, value)
await self.conn.indices.open(real_index_name_next_version)
# Start to duplicate aliases
body = {
"actions": [
{"remove": {
"alias": index_name,
"index": real_index_name
}},
{"add": {
"alias": index_name,
"index": temp_index
}}
]
}
async with conn_es._session.post(
conn_es._base_url + '_aliases',
data=json.dumps(body),
timeout=1000000
) as resp:
pass
logger.warn('Updated aliases')
# Reindex
body = {
"source": {
"index": real_index_name
},
"dest": {
"index": real_index_name_next_version
}
}
conn_es = await self.conn.transport.get_connection()
async with conn_es._session.post(
conn_es._base_url + '_reindex',
data=json.dumps(body),
timeout=10000000
) as resp:
pass
logger.warn('Reindexed')
# Move aliases
body = {
"actions": [
{"remove": {
"alias": index_name,
"index": temp_index
}},
{"add": {
"alias": index_name,
"index": real_index_name_next_version
}}
]
}
conn_es = await self.conn.transport.get_connection()
async with conn_es._session.post(
conn_es._base_url + '_aliases',
data=json.dumps(body),
timeout=1000000
) as resp:
pass
logger.warn('Updated aliases')
self.set_version(site, next_version)
# Reindex
body = {
"source": {
"index": temp_index
},
"dest": {
"index": real_index_name_next_version
}
}
async with conn_es._session.post(
conn_es._base_url + '_reindex',
data=json.dumps(body)
) as resp:
pass
logger.warn('Reindexed temp')
# Delete old index
await self.conn.indices.close(real_index_name)
await self.conn.indices.delete(real_index_name)
await self.conn.indices.close(temp_index)
await self.conn.indices.delete(temp_index)
def settings(self):
settings = app_settings.get('mailer', {})
settings.update(self._settings.get('mailer', {}))
return settings
def make_app(config_file=None, settings=None):
app_settings.update(_delayed_default_settings)
# Initialize aiohttp app
app = web.Application(router=TraversalRouter())
# Create root Application
root = ApplicationRoot(config_file)
root.app = app
provideUtility(root, IApplication, 'root')
# Initialize global (threadlocal) ZCA configuration
app.config = ConfigurationMachine()
registerCommonDirectives(app.config)
if config_file is not None:
with open(config_file, 'r') as config:
settings = json.load(config)
elif settings is None:
raise Exception('Neither configuration or settings')
import plone.server
configure.include("zope.component")
configure.include("zope.annotation")
configure.include("plone.server", "meta.zcml") # bbb
configure.scan('plone.server.translation')
configure.scan('plone.server.renderers')
configure.scan('plone.server.api')
configure.scan('plone.server.content')
configure.scan('plone.server.security')
configure.scan('plone.server.json')
configure.scan('plone.server.behaviors')
configure.scan('plone.server.languages')
configure.scan('plone.server.permissions')
configure.scan('plone.server.migrate.migrations')
configure.scan('plone.server.auth.participation')
configure.scan('plone.server.auth.principalrole')
configure.scan('plone.server.catalog.index')
configure.scan('plone.server.catalog.catalog')
configure.scan('plone.server.framing')
configure.scan('plone.server.file')
configure.scan('plone.server.types')
load_application(plone.server, root, settings)
for ep in iter_entry_points('plone.server'):
# auto-include applications
# What an "app" include consists of...
# 1. load zcml if present
# 2. load "includeme" module function if present
# 3. load app_settings dict if present in the module
if ep.module_name not in settings.get('applications', []):
continue
load_application(ep.load(), root, settings)
try:
app.config.execute_actions()
except ConfigurationConflictError as e:
logger.error(str(e._conflicts))
raise e
# XXX we clear now to save some memory
# it's unclear to me if this is necesary or not but it seems to me that
# we don't need things registered in both components AND here.
configure.clear()
# update *after* plugins loaded
update_app_settings(settings)
content_type = ContentNegotiatorUtility(
'content_type', app_settings['renderers'].keys())
language = ContentNegotiatorUtility(
'language', app_settings['languages'].keys())
provideUtility(content_type, IContentNegotiation, 'content_type')
provideUtility(language, IContentNegotiation, 'language')
for database in app_settings['databases']:
for key, dbconfig in database.items():
factory = getUtility(
IDatabaseConfigurationFactory, name=dbconfig['storage'])
root[key] = factory(key, dbconfig)
for static in app_settings['static']:
for key, file_path in static.items():
root[key] = StaticFile(file_path)
root.set_root_user(app_settings['root_user'])
if RSA is not None and not app_settings.get('rsa'):
key = RSA.generate(2048)
pub_jwk = {'k': key.publickey().exportKey('PEM')}
priv_jwk = {'k': key.exportKey('PEM')}
app_settings['rsa'] = {
'pub': pub_jwk,
'priv': priv_jwk
}
# Set router root
app.router.set_root(root)
for utility in getAllUtilitiesRegisteredFor(IAsyncUtility):
# In case there is Utilties that are registered from zcml
ident = asyncio.ensure_future(utility.initialize(app=app), loop=app.loop)
root.add_async_utility(ident, {})
app.on_cleanup.append(close_utilities)
for util in app_settings['utilities']:
root.add_async_utility(util)
# Load cached Schemas
load_cached_schema()
return app
def update_app_settings(settings):
for key, value in settings.items():
if isinstance(app_settings.get(key), dict):
app_settings[key].update(value)
else:
app_settings[key] = value
def make_app(config_file=None, settings=None):
app_settings.update(_delayed_default_settings)
# Initialize aiohttp app
app = web.Application(
router=TraversalRouter(),
**settings.get('aiohttp_settings', {}))
# Create root Application
root = ApplicationRoot(config_file)
root.app = app
provideUtility(root, IApplication, 'root')
# Initialize global (threadlocal) ZCA configuration
app.config = ConfigurationMachine()
registerCommonDirectives(app.config)
if config_file is not None:
with open(config_file, 'r') as config:
settings = json.load(config)
elif settings is None:
raise Exception('Neither configuration or settings')
import plone.server
configure.include("zope.component")
configure.include("zope.annotation")
configure.include("plone.server", "meta.zcml") # bbb
configure.scan('plone.server.translation')
configure.scan('plone.server.renderers')
configure.scan('plone.server.api')
configure.scan('plone.server.content')
configure.scan('plone.server.auth')
configure.scan('plone.server.json')
configure.scan('plone.server.behaviors')
configure.scan('plone.server.languages')
configure.scan('plone.server.permissions')
configure.scan('plone.server.migrate.migrations')
configure.scan('plone.server.auth.checker')
configure.scan('plone.server.auth.security_local')
configure.scan('plone.server.auth.policy')
configure.scan('plone.server.auth.participation')
configure.scan('plone.server.catalog.index')
configure.scan('plone.server.catalog.catalog')
configure.scan('plone.server.framing')
configure.scan('plone.server.file')
configure.scan('plone.server.types')
load_application(plone.server, root, settings)
for ep in iter_entry_points('plone.server'):
# auto-include applications
# What an "app" include consists of...
# 1. load zcml if present
# 2. load "includeme" module function if present
# 3. load app_settings dict if present in the module
if ep.module_name not in settings.get('applications', []):
continue
load_application(ep.load(), root, settings)
try:
app.config.execute_actions()
except ConfigurationConflictError as e:
logger.error(str(e._conflicts))
raise e
# XXX we clear now to save some memory
# it's unclear to me if this is necesary or not but it seems to me that
# we don't need things registered in both components AND here.
configure.clear()
# update *after* plugins loaded
update_app_settings(settings)
content_type = ContentNegotiatorUtility(
'content_type', app_settings['renderers'].keys())
language = ContentNegotiatorUtility(
'language', app_settings['languages'].keys())
provideUtility(content_type, IContentNegotiation, 'content_type')
provideUtility(language, IContentNegotiation, 'language')
for database in app_settings['databases']:
for key, dbconfig in database.items():
factory = getUtility(
IDatabaseConfigurationFactory, name=dbconfig['storage'])
root[key] = factory(key, dbconfig)
for static in app_settings['static']:
for key, file_path in static.items():
path = pathlib.Path(file_path)
if path.is_dir():
root[key] = StaticDirectory(path)
else:
root[key] = StaticFile(path)
root.set_root_user(app_settings['root_user'])
if RSA is not None and not app_settings.get('rsa'):
key = RSA.generate(2048)
pub_jwk = {'k': key.publickey().exportKey('PEM')}
priv_jwk = {'k': key.exportKey('PEM')}
app_settings['rsa'] = {
'pub': pub_jwk,
'priv': priv_jwk
}
# Set router root
app.router.set_root(root)
for utility in getAllUtilitiesRegisteredFor(IAsyncUtility):
# In case there is Utilties that are registered from zcml
ident = asyncio.ensure_future(utility.initialize(app=app), loop=app.loop)
root.add_async_utility(ident, {})
app.on_cleanup.append(close_utilities)
for util in app_settings['utilities']:
root.add_async_utility(util)
# Load cached Schemas
load_cached_schema()
return app
def update_app_settings(settings):
for key, value in settings.items():
if isinstance(app_settings.get(key), dict):
app_settings[key].update(value)
else:
app_settings[key] = value