def __init__(self, store, filehandle=sys.stdout, config=None, filter_use=None): """Initializes the Elastic output module.""" super(Elastic, self).__init__(store, filehandle, config, filter_use) self._counter = 0 self._data = [] # TODO: move this to an output module interface. self._formatters_manager = formatters_manager.EventFormatterManager elastic_host = getattr(config, 'elastic_server', '127.0.0.1') elastic_port = getattr(config, 'elastic_port', 9200) self._elastic_db = pyelasticsearch.ElasticSearch( u'http://{0:s}:{1:d}'.format(elastic_host, elastic_port)) case_name = getattr(config, 'case_name', u'') document_type = getattr(config, 'document_type', u'') # case_name becomes the index name in Elastic. if case_name: self._index_name = case_name.lower() else: self._index_name = uuid.uuid4().hex # Name of the doc_type that holds the plaso events. if document_type: self._doc_type = document_type.lower() else: self._doc_type = u'event' # Build up a list of available hostnames in this storage file. self._hostnames = {} self._preprocesses = {}
def setUp(self): super(ElasticsearchBoostBackendTestCase, self).setUp() # Wipe it clean. self.raw_es = pyelasticsearch.ElasticSearch(settings.HAYSTACK_CONNECTIONS['default']['URL']) clear_elasticsearch_index() # Stow. self.old_ui = connections['default'].get_unified_index() self.ui = UnifiedIndex() self.smmi = ElasticsearchBoostMockSearchIndex() self.ui.build(indexes=[self.smmi]) connections['default']._index = self.ui self.sb = connections['default'].get_backend() self.sample_objs = [] for i in xrange(1, 5): mock = AFourthMockModel() mock.id = i if i % 2: mock.author = 'daniel' mock.editor = 'david' else: mock.author = 'david' mock.editor = 'daniel' mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) self.sample_objs.append(mock)
def __init__(self, output_mediator, **kwargs): """Initializes the output module object. Args: output_mediator: The output mediator object (instance of OutputMediator). """ super(ElasticSearchOutputModule, self).__init__(output_mediator, **kwargs) self._counter = 0 self._data = [] elastic_host = self._output_mediator.GetConfigurationValue( u'elastic_server', default_value=u'127.0.0.1') elastic_port = self._output_mediator.GetConfigurationValue( u'elastic_port', default_value=9200) self._elastic_db = pyelasticsearch.ElasticSearch( u'http://{0:s}:{1:d}'.format(elastic_host, elastic_port)) case_name = self._output_mediator.GetConfigurationValue( u'case_name', default_value=u'') document_type = self._output_mediator.GetConfigurationValue( u'document_type', default_value=u'') # case_name becomes the index name in Elastic. if case_name: self._index_name = case_name.lower() else: self._index_name = uuid.uuid4().hex # Name of the doc_type that holds the plaso events. if document_type: self._doc_type = document_type.lower() else: self._doc_type = u'event'
def search(self, query, size=1000, recent=False): """Search an elasticsearch server. `query` parameter is the complicated query structure that pyelasticsearch uses. More details in their documentation. `size` is the max number of results to return from the search engine. We default it to 1000 to ensure we don't loose things. For certain classes of queries (like faceted ones), this can actually be set very low, as it won't impact the facet counts. `recent` search only most recent indexe(s), assuming this is basically a real time query that you only care about the last hour of time. Using recent dramatically reduces the load on the ES cluster. The returned result is a ResultSet query. """ es = pyelasticsearch.ElasticSearch(self._url) args = {'size': size} if recent: # today's index datefmt = 'logstash-%Y.%m.%d' now = datetime.datetime.utcnow() lasthr = now - datetime.timedelta(hours=1) indexes = [now.strftime(datefmt)] if (lasthr.strftime(datefmt) != now.strftime(datefmt)): indexes.append(lasthr.strftime(datefmt)) args['index'] = indexes results = es.search(query, **args) return ResultSet(results)
def setUp(self): super(ElasticsearchSearchBackendTestCase, self).setUp() # Wipe it clean. self.raw_es = pyelasticsearch.ElasticSearch(settings.HAYSTACK_CONNECTIONS['default']['URL']) clear_elasticsearch_index() # Stow. self.old_ui = connections['default'].get_unified_index() self.ui = UnifiedIndex() self.smmi = ElasticsearchMockSearchIndex() self.smtmmi = ElasticsearchMaintainTypeMockSearchIndex() self.ui.build(indexes=[self.smmi]) connections['default']._index = self.ui self.sb = connections['default'].get_backend() # Force the backend to rebuild the mapping each time. self.sb.existing_mapping = {} self.sb.setup() self.sample_objs = [] for i in xrange(1, 4): mock = MockModel() mock.id = i mock.author = 'daniel%s' % i mock.pub_date = datetime.date(2009, 2, 25) - datetime.timedelta(days=i) self.sample_objs.append(mock)
def __init__(self, settings=None): self.settings = settings or { "number_of_shards": 1, "number_of_replicas": 0 } self.client = es.ElasticSearch(urls=["http://localhost:9200"]) self.index_name = None self._buffer = []
def clear_elasticsearch_index(): # Wipe it clean. raw_es = pyelasticsearch.ElasticSearch(settings.HAYSTACK_CONNECTIONS['default']['URL']) try: raw_es.delete_index(settings.HAYSTACK_CONNECTIONS['default']['INDEX_NAME']) raw_es.refresh() except (requests.RequestException, pyelasticsearch.ElasticHttpError): pass
def recreate_index(): index = "acousticbrainz" es = pyelasticsearch.ElasticSearch(config.ELASTICSEARCH_ADDRESS) try: es.delete_index(index) except pyelasticsearch.ElasticHttpNotFoundError: pass es.create_index(index)
def SetServerInformation(self, elastic_host, elastic_port): """Set the ElasticSearch connection. Args: elastic_host: the hostname or IP address of the ElasticSearch server. elastic_port: the port number that the ElasticSearch is listening on. """ self._elastic_db = pyelasticsearch.ElasticSearch( u'http://{0:s}:{1:d}'.format(elastic_host, elastic_port))
def _configure_es(host, timeout): import pyelasticsearch global elasticsearch try: elasticsearch = pyelasticsearch.ElasticSearch(host, timeout=timeout, max_retries=0) except Exception as e: elasticsearch = ErrorProxy(e)
def __init__(self, config, quit_check_callback=None): super(ElasticSearchCrashStorage, self).__init__(config, quit_check_callback) self.transaction = config.transaction_executor_class( config, self, quit_check_callback) if self.config.elasticsearch_urls: self.es = pyelasticsearch.ElasticSearch( self.config.elasticsearch_urls, timeout=self.config.elasticsearch_timeout) else: config.logger.warning('elasticsearch crash storage is disabled.')
def _check_er_availability(): global es_url global query_dir if not classifier: if not er: health = 'NotInstalled' elif not es_url or not query_dir: health = 'NotConfigured' else: url = classifier.config.es_url es = pyelasticsearch.ElasticSearch(url) health = {'Configured': {'elastic-search': es.health()['status']}} return health
def __init__(self, store, formatter_mediator, filehandle=sys.stdout, config=None, filter_use=None): """Initializes the log output formatter object. Args: store: A storage file object (instance of StorageFile) that defines the storage. formatter_mediator: the formatter mediator object (instance of FormatterMediator). filehandle: Optional file-like object that can be written to. The default is sys.stdout. config: Optional configuration object, containing config information. The default is None. filter_use: Optional filter object (instance of FilterObject). The default is None. """ super(ElasticSearchOutput, self).__init__(store, formatter_mediator, filehandle=filehandle, config=config, filter_use=filter_use) self._counter = 0 self._data = [] elastic_host = getattr(config, 'elastic_server', '127.0.0.1') elastic_port = getattr(config, 'elastic_port', 9200) self._elastic_db = pyelasticsearch.ElasticSearch( u'http://{0:s}:{1:d}'.format(elastic_host, elastic_port)) case_name = getattr(config, 'case_name', u'') document_type = getattr(config, 'document_type', u'') # case_name becomes the index name in Elastic. if case_name: self._index_name = case_name.lower() else: self._index_name = uuid.uuid4().hex # Name of the doc_type that holds the plaso events. if document_type: self._doc_type = document_type.lower() else: self._doc_type = u'event' # Build up a list of available hostnames in this storage file. self._hostnames = {} self._preprocesses = {}
def get_s3_bucket_dir_to_index(self): if len(self.path_name_s3_billing) == 1: prefix = '/' + '/'.join(self.path_name_s3_billing) + '/' else: prefix = '/'.join(self.path_name_s3_billing) + '/' key_names = self.s3.list_objects(Bucket=self.bucketname, Prefix=prefix, Delimiter='/') s3_dir_names = [] if 'CommonPrefixes' not in key_names: return 1 for keys in key_names['CommonPrefixes']: s3_dir_names.append(keys['Prefix'].split('/')[-2]) s3_dir_names.sort() es = pyes.ElasticSearch('http://*****:*****@timestamp']) index_time.sort(reverse=True) dir_start = 0 dir_end = None if index_time: current_dir = dtd.today().strftime('%Y%m01') + '-' + (dtd.today() + \ relativedelta(months=1)).strftime('%Y%m01') last_ind_dir = index_time[0].split('T')[0].replace('-', '') last_ind_dir = dtdt.strptime(last_ind_dir, '%Y%m%d').strftime( '%Y%m01') + '-' + (dtdt.strptime(last_ind_dir, '%Y%m%d') + relativedelta(months=1)).strftime('%Y%m01') dir_start = s3_dir_names.index(last_ind_dir) dir_end = s3_dir_names.index(current_dir) + 1 s3_dir_to_index = s3_dir_names[dir_start:dir_end] print('Months to be indexed: {}'.format(', '.join(s3_dir_to_index))) # returning only the dirnames which are to be indexed return s3_dir_to_index
def __init__(self, connection_alias, **connection_options): super(ElasticsearchSearchBackend, self).__init__(connection_alias, **connection_options) if not 'URL' in connection_options: raise ImproperlyConfigured("You must specify a 'URL' in your settings for connection '%s'." % connection_alias) if not 'INDEX_NAME' in connection_options: raise ImproperlyConfigured("You must specify a 'INDEX_NAME' in your settings for connection '%s'." % connection_alias) self.conn = pyelasticsearch.ElasticSearch(connection_options['URL'], timeout=self.timeout) self.index_name = connection_options['INDEX_NAME'] self.log = logging.getLogger('haystack') self.setup_complete = False self.existing_mapping = {}
def get(self, **kwargs): '''Return the result of a custom query. ''' params = external_common.parse_arguments(self.filters, kwargs) if not params.query: raise MissingArgumentError('query') try: query = json.loads(params.query) except ValueError: raise BadArgumentError( 'query', msg="Invalid JSON value for parameter 'query'" ) es = pyelasticsearch.ElasticSearch( urls=self.config.elasticsearch_urls, timeout=self.config.elasticsearch_timeout_extended, ) # Set indices. indices = [] if not params.indices: # By default, use the last two indices. today = utc_now() last_week = today - datetime.timedelta(days=7) indices = self.generate_list_of_indexes(last_week, today) elif len(params.indices) == 1 and params.indices[0] == 'ALL': # If we want all indices, just do nothing. pass else: indices = params.indices search_args = {} if indices: search_args['index'] = indices search_args['doc_type'] = self.config.elasticsearch_doctype try: results = es.search( query, **search_args ) except ElasticHttpNotFoundError, e: missing_index = re.findall(BAD_INDEX_REGEX, e.error)[0] raise ResourceNotFound( "elasticsearch index '%s' does not exist" % missing_index )
def __init__(self, config, quit_check_callback=None): super(ElasticSearchCrashStorage, self).__init__(config, quit_check_callback) self.transaction = config.transaction_executor_class( config, self, quit_check_callback) if self.config.elasticsearch_urls: self.es = pyelasticsearch.ElasticSearch( self.config.elasticsearch_urls, timeout=self.config.timeout) settings_json = open( self.config.elasticsearch_index_settings).read() self.index_settings = json.loads(settings_json % self.config.elasticsearch_doctype) else: config.logger.warning('elasticsearch crash storage is disabled.')
def search(self, query, size=1000, recent=False, days=0): """Search an elasticsearch server. `query` parameter is the complicated query structure that pyelasticsearch uses. More details in their documentation. `size` is the max number of results to return from the search engine. We default it to 1000 to ensure we don't loose things. For certain classes of queries (like faceted ones), this can actually be set very low, as it won't impact the facet counts. `recent` search only most recent indexe(s), assuming this is basically a real time query that you only care about the last hour of time. Using recent dramatically reduces the load on the ES cluster. `days` search only the last number of days. The returned result is a ResultSet query. """ es = pyelasticsearch.ElasticSearch(self._url) args = {'size': size} if recent or days: # today's index datefmt = self._indexfmt now = datetime.datetime.utcnow() indexes = [] latest_index = now.strftime(datefmt) if self._is_valid_index(es, latest_index): indexes.append(latest_index) if recent: lasthr = now - datetime.timedelta(hours=1) lasthr_index = lasthr.strftime(datefmt) if lasthr_index != latest_index: if self._is_valid_index(es, lasthr_index): indexes.append(lasthr.strftime(datefmt)) for day in range(1, days): lastday = now - datetime.timedelta(days=day) index_name = lastday.strftime(datefmt) if self._is_valid_index(es, index_name): indexes.append(index_name) args['index'] = indexes results = es.search(query, **args) return ResultSet(results)
def main(self): storage = self.config.elasticsearch_storage_class(self.config) crash_file = open(self.config.processed_crash_file) processed_crash = json.load(crash_file) es_index = storage.get_index_for_crash(processed_crash) es_doctype = self.config.elasticsearch_doctype crash_id = processed_crash['uuid'] storage.save_processed(processed_crash) # Verify the crash has been inserted es = pyelasticsearch.ElasticSearch(self.config.elasticsearch_urls) crash = es.get(es_index, es_doctype, crash_id) assert crash['exists'] print 'Success - %s/%s/%s' % (es_index, es_doctype, crash_id)
def delete_old_indices(self): now = utc_now() policy_delay = datetime.timedelta(weeks=self.config.retention_policy) time_limit = (now - policy_delay).replace(tzinfo=None) es = pyelasticsearch.ElasticSearch( self.config.elasticsearch.elasticsearch_urls, timeout=self.config.elasticsearch.elasticsearch_timeout ) state = es.cluster_state() indices = state['metadata']['indices'].keys() aliases = es.aliases() for index in indices: # Some indices look like 'socorro%Y%W_%Y%M%d', but they are # aliased to the expected format of 'socorro%Y%W'. In such cases, # replace the index with the alias. if index in aliases: index_aliases = aliases[index]['aliases'].keys() if index_aliases: index = index_aliases[0] if not re.match( self.config.elasticsearch.elasticsearch_index_regex, index ): # This index doesn't look like a crash index, let's skip it. continue # This won't take the week part of our indices into account... index_date = datetime.datetime.strptime( index, self.config.elasticsearch.elasticsearch_index ) # So we need to get that differently, and then add it to the date. index_date += datetime.timedelta(weeks=int(index[-2:])) if index_date < time_limit: es.delete_index(index) # Bad index! Go away!
def get_connection(): return pyelasticsearch.ElasticSearch(settings.ELASTICSEARCH_URL)
import re import config # Setup Flask app = Flask(__name__) # Setup Mongo conn = pymongo.Connection() # defaults to localhost db = conn.agile_data emails = db['emails'] addresses_per_email = db['addresses_per_email'] emails_per_address = db['emails_per_address'] sent_distributions = db['sent_distributions'] # Setup ElasticSearch elastic = pyelasticsearch.ElasticSearch(config.ELASTIC_URL) # Controller: Fetch an email and display it @app.route("/email/<message_id>") def email(message_id): email = emails.find_one({'message_id': message_id}) addresses = addresses_per_email.find_one({'message_id': message_id}) sent_dist_records = sent_distributions.find_one( {'address': email['from']['address']}) return render_template('partials/email.html', email=email, addresses=addresses['addresses'], chart_json=json.dumps( sent_dist_records['sent_distribution']), sent_distribution=sent_dist_records)
import pyelasticsearch as pyes keyword = '張世瑛' es = pyes.ElasticSearch('http://localhost:9200') # query = {'query':{'match':{'message':'資訊部'}}} # query = {'query':{'match_phrase':{'message':'資訊部'}}} # query = {'query':{'match_phrase':{'comments.data.message':'鄧美玉'}}} # query = { # 'filtered':{ # 'query':{ # 'multi_match':{ # 'type':'phrase', # 'query':keyword, # 'fields':['comments.data.message','message'] # } # }, # 'filter':{ # 'and':[ # {'match_phrase':{'message':'會計學'}}, # {'match_phrase':{'message':'鄧美玉'}} # ] # } # } # } query = { 'query': { 'multi_match': {
import pyelasticsearch es = pyelasticsearch.ElasticSearch("http://localhost:9200") def autocomplete_genre(g): query = {"placeholder": {"text": g.lower(), "completion": {"field": "toptags.tag.name_complete"}}} completes = es._search_or_count('_suggest', query, index="acousticbrainz") if "placeholder" in completes: options = completes["placeholder"][0]["options"] options = sorted(options, key=lambda x: x["score"]) else: options = [] return [o["text"] for o in options] def autocomplete_artist(g): query = {"placeholder": {"text": g.lower(), "completion": {"field": "metadata.tags.artist_complete"}}} completes = es._search_or_count('_suggest', query, index="acousticbrainz") if "placeholder" in completes: options = completes["placeholder"][0]["options"] options = sorted(options, key=lambda x: x["score"]) else: options = [] return [o["text"] for o in options] def autocomplete_track(t): query = {"placeholder": {"text": t.lower(), "completion": {"field": "metadata.tags.title_complete"}}} completes = es._search_or_count('_suggest', query, index="acousticbrainz") if "placeholder" in completes: options = completes["placeholder"][0]["options"] options = sorted(options, key=lambda x: x["score"]) else:
logger = logging.getLogger('z.elasticsearch') # Enable these to get full debugging information. # logging.getLogger('pyelasticsearch').setLevel(logging.DEBUG) # logging.getLogger('requests').setLevel(logging.DEBUG) # The subset of settings.ES_INDEXES we are concerned with. ALIAS = settings.ES_INDEXES['webapp'] if hasattr(settings, 'ES_URLS'): ES_URL = settings.ES_URLS[0] else: ES_URL = 'http://127.0.0.1:9200' ES = pyelasticsearch.ElasticSearch(ES_URL) job = 'lib.es.management.commands.reindex_mkt.run_indexing' time_limits = settings.CELERY_TIME_LIMITS[job] @task def delete_index(old_index): """Removes the index.""" sys.stdout.write('Removing index %r' % old_index) ES.delete_index(old_index) @task def create_index(new_index, alias, settings): """Creates a mapping for the new index.
def __init__(self, urls, settings, *args, **kwargs): self._default_settings = settings self._es = pyelasticsearch.ElasticSearch(urls, *args, **kwargs)
sys.path.append("../") import os #os.environ.setdefault("DJANGO_SETTINGS_MODULE", "beCOMPANY.settings") #import beCOMPANY #import beCOMPANY.settings as settings from elasticmodel import * from dateutil.parser import parse #import psycopg2 import threading from dateutil.relativedelta import relativedelta import time import pyelasticsearch from pyelasticsearch import bulk_chunks es = pyelasticsearch.ElasticSearch(port=9200) eastern = timezone('US/Eastern') ''' try: dbstr="dbname=" + settings.DATABASES['default']['NAME'] + \ " user="******" password="******" host=" + settings.DATABASES['default']['HOST'] + \ " port=" + settings.DATABASES['default']['PORT'] c=psycopg2.connect(dbstr) except: pass #print "I am unable to connect to the database." ''' Instrument.init() Feed.init()
Search module for the scrAPI website. """ import logging import pyelasticsearch import search_settings logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) # These are the doc_types that exist in the search database TYPES = ['article', 'citation'] try: elastic = pyelasticsearch.ElasticSearch( search_settings.ELASTIC_URI, timeout=search_settings.ELASTIC_TIMEOUT ) logging.getLogger('pyelasticsearch').setLevel(logging.WARN) logging.getLogger('requests').setLevel(logging.WARN) elastic.health() except pyelasticsearch.exceptions.ConnectionError as e: logger.error(e) logger.warn("The SEARCH_ENGINE setting is set to 'elastic', but there " "was a problem starting the elasticsearch interface. Is " "elasticsearch running?") elastic = None def requires_search(func): def wrapped(*args, **kwargs): if elastic is not None:
from the data folder ''' dataFolder = "../data/" #=========================================================================== # THIS IS ONE WAY TO INDEX. there seems to be a limit of 1k fields per index. Which is generous, # but surprisingly one csv file had more than that ?? # also is this limit a lucene/es limit, or the client limit. Does the "official client improve this" #=========================================================================== #csv2es --index-name potatoes --doc-type potato --import-file potatoes.cs ''' pip-install csv2es pip-install pyelasticsearch ''' es_client = pyelasticsearch.ElasticSearch('http://localhost:9200/') myDocuments = csv2es.documents_from_file(es_client, "./test.csv", ",", quiet = False) csv2es.perform_bulk_index(host = 'http://localhost:9200/' , index_name = "baseindex", doc_type = "basedoctype", doc_fetch = myDocuments, docs_per_chunk = 5000, bytes_per_chunk = 100000, parallel = 1) print("END of elastic search test script")
def main(global_config, **settings): """ This function returns a Pyramid WSGI application. """ auth_tkt_policy = AuthTktAuthenticationPolicy( settings['authtkt.secret'], hashalg='sha512', callback=groupfinder, max_age=2592000, secure=asbool(settings.get('authtkt.secure', 'false'))) auth_token_policy = AuthTokenAuthenticationPolicy( callback=groupfinder ) authorization_policy = ACLAuthorizationPolicy() authentication_policy = AuthenticationStackPolicy() authentication_policy.add_policy('auth_tkt', auth_tkt_policy) authentication_policy.add_policy('auth_token', auth_token_policy) # set crypto key encryption.ENCRYPTION_SECRET = settings.get('encryption_secret') # import this later so encyption key can be monkeypatched from appenlight.models import DBSession, register_datastores # update config with cometd info settings['cometd_servers'] = {'server': settings['cometd.server'], 'secret': settings['cometd.secret']} # Create the Pyramid Configurator. settings['_mail_url'] = settings['mailing.app_url'] config = CythonCompatConfigurator( settings=settings, authentication_policy=authentication_policy, authorization_policy=authorization_policy, root_factory='appenlight.security.RootFactory', default_permission='view') # custom registry variables # resource type information config.registry.resource_types = ['resource', 'application'] # plugin information config.registry.appenlight_plugins = {} config.set_default_csrf_options(require_csrf=True, header='X-XSRF-TOKEN') config.add_view_deriver('appenlight.predicates.csrf_view', name='csrf_view') # later, when config is available dogpile_config = {'url': settings['redis.url'], "redis_expiration_time": 86400, "redis_distributed_lock": True} cache_regions.regions = cache_regions.CacheRegions(dogpile_config) config.registry.cache_regions = cache_regions.regions engine = engine_from_config(settings, 'sqlalchemy.', json_serializer=json.dumps) DBSession.configure(bind=engine) # json rederer that serializes datetime config.add_renderer('json', json_renderer) config.set_request_property('appenlight.lib.request.es_conn', 'es_conn') config.set_request_property('appenlight.lib.request.get_user', 'user', reify=True) config.set_request_property('appenlight.lib.request.get_csrf_token', 'csrf_token', reify=True) config.set_request_property('appenlight.lib.request.safe_json_body', 'safe_json_body', reify=True) config.set_request_property('appenlight.lib.request.unsafe_json_body', 'unsafe_json_body', reify=True) config.add_request_method('appenlight.lib.request.add_flash_to_headers', 'add_flash_to_headers') config.add_request_method('appenlight.lib.request.get_authomatic', 'authomatic', reify=True) config.include('pyramid_redis_sessions') config.include('pyramid_tm') config.include('pyramid_jinja2') config.include('appenlight_client.ext.pyramid_tween') config.include('ziggurat_foundations.ext.pyramid.sign_in') es_server_list = aslist(settings['elasticsearch.nodes']) redis_url = settings['redis.url'] log.warning('Elasticsearch server list: {}'.format(es_server_list)) log.warning('Redis server: {}'.format(redis_url)) config.registry.es_conn = pyelasticsearch.ElasticSearch(es_server_list) config.registry.redis_conn = redis.StrictRedis.from_url(redis_url) config.registry.redis_lockmgr = Redlock([settings['redis.redlock.url'], ], retry_count=0, retry_delay=0) # mailer config.registry.mailer = Mailer.from_settings(settings) # Configure sessions session_factory = session_factory_from_settings(settings) config.set_session_factory(session_factory) # Configure renderers and event subscribers config.add_jinja2_extension('jinja2.ext.loopcontrols') config.add_jinja2_search_path('appenlight:templates') # event subscribers config.add_subscriber("appenlight.subscribers.application_created", "pyramid.events.ApplicationCreated") config.add_subscriber("appenlight.subscribers.add_renderer_globals", "pyramid.events.BeforeRender") config.add_subscriber('appenlight.subscribers.new_request', 'pyramid.events.NewRequest') config.add_view_predicate('context_type_class', 'appenlight.predicates.contextTypeClass') register_datastores(es_conn=config.registry.es_conn, redis_conn=config.registry.redis_conn, redis_lockmgr=config.registry.redis_lockmgr) # base stuff and scan # need to ensure webassets exists otherwise config.override_asset() # throws exception if not os.path.exists(settings['webassets.dir']): os.mkdir(settings['webassets.dir']) config.add_static_view(path='appenlight:webassets', name='static', cache_max_age=3600) config.override_asset(to_override='appenlight:webassets/', override_with=settings['webassets.dir']) config.include('appenlight.views') config.include('appenlight.views.admin') config.scan(ignore=['appenlight.migrations', 'appenlight.scripts', 'appenlight.tests']) config.add_directive('register_appenlight_plugin', register_appenlight_plugin) for entry_point in iter_entry_points(group='appenlight.plugins'): plugin = entry_point.load() plugin.includeme(config) # include other appenlight plugins explictly if needed includes = aslist(settings.get('appenlight.includes', [])) for inc in includes: config.include(inc) # run this after everything registers in configurator def pre_commit(): jinja_env = config.get_jinja2_environment() jinja_env.filters['tojson'] = json.dumps jinja_env.filters['toJSONUnsafe'] = jinja2_filters.toJSONUnsafe config.action(None, pre_commit, order=PHASE3_CONFIG + 999) def wrap_config_celery(): configure_celery(config.registry) config.action(None, wrap_config_celery, order=PHASE3_CONFIG + 999) app = config.make_wsgi_app() return app