def mock_client():
client = Mock()
client.search.return_value = dummy_response()
connections.add_connection('mock', client)
yield client
connections._conn = {}
connections._kwargs = {}
def setUp(self):
super().setUp()
self.es = search.init_es()
connections.add_connection('default', self.es)
self.s = Search(index=settings.ELASTICSEARCH_INDEX)
search.Image.init()
self.es.cluster.health(wait_for_status='yellow', request_timeout=2000)
self.img1 = models.Image(title='greyhounds are fast',
creator="Rashid",
url='http://example.com/1',
license='CC0',
provider="flickr",
source="openimages",
tags_list=['greyhound', 'dog', 'object'])
self.img2 = models.Image(title='pumpkins are orange',
creator='諸葛亮',
url='http://example.com/2',
license='CC-BY',
provider="rijksmuseum",
source="rijksmuseum",
tags_list=['gourds', 'fruit', 'object'])
self.img1.save()
self.img2.save()
self.url = reverse('index')
self.removed = models.Image.objects.create(title='removed',
url=FOREIGN_URL +
TEST_IMAGE_REMOVED,
license="cc0")
def mock_client(dummy_response):
client = Mock()
client.search.return_value = dummy_response
connections.add_connection('mock', client)
yield client
connections._conn = {}
connections._kwargs = {}
def client():
try:
connection = get_test_client(nowait='WAIT_FOR_ES' not in os.environ)
connections.add_connection('default', connection)
return connection
except SkipTest:
skip()
def get_top_k(text_query):
text_query = ' '.join(pre_process_text(text_query))
es = Elasticsearch()
connections.add_connection('CivilArticle', es)
es_index = 'coliee_bm25_index'
es_query = {
"multi_match": {
"query": text_query,
"fields": ["title", "content"]
}
}
res = es.search(index=es_index,
body={
"from": 0,
"size": top_k,
"query": es_query
})
res = res['hits']['hits']
articles_scores = [{
'article': article['_source']["code"].lower(),
'score': article['_score']
} for article in res]
return articles_scores
def client():
try:
connection = get_test_client(nowait='WAIT_FOR_ES' not in os.environ)
connections.add_connection('default', connection)
return connection
except SkipTest:
skip()
def init(timeout=TIMEOUT):
"""Initialize all search objects"""
es = init_es(timeout=timeout)
connections.add_connection('default', es)
log.debug("Initializing search objects for connection %s:%s",
settings.ELASTICSEARCH_URL, settings.ELASTICSEARCH_PORT)
return es
def configure_es_from_config(settings):
global elasticsearch_config
client = client_from_config(settings)
connections.add_connection('default', client)
elasticsearch_config['client'] = client
elasticsearch_config['index'] = settings['elasticsearch.index']
elasticsearch_config['host'] = settings['elasticsearch.host']
elasticsearch_config['port'] = int(settings['elasticsearch.port'])
def es_client():
# NOTE (fschmidt): I took a look on how elasticsearch-dsl does the mocking:
# https://github.com/elastic/elasticsearch-dsl-py/blob/master/test_elasticsearch_dsl/conftest.py
client = mock.Mock()
client.search.return_value = {}
connections.add_connection("default", client)
yield client
connections._conn = {}
connections._kwargs = {}
def mock_client(request):
# inner import to avoid throwing off coverage
from elasticsearch_dsl.connections import connections
client = Mock()
client.search.return_value = dummy_response()
connections.add_connection('mock', client)
yield client
connections._conn = {}
connections._kwargs = {}
def mock_es_client(request):
"""
Mock ElasticSearch client.
User should override client.search.return_value.
Based on test fixture from elasticsearch_dsl
"""
client = mock.Mock()
connections.add_connection('default', client)
yield client
connections._conn = {}
connections._kwargs = {}
def mock_es_client(request):
"""
Mock ElasticSearch client.
User should override client.search.return_value.
Based on test fixture from elasticsearch_dsl
"""
client = mock.Mock()
connections.add_connection('default', client)
yield client
connections._conn = {}
connections._kwargs = {}
def mock_client(request):
# inner import to avoid throwing off coverage
from elasticsearch_dsl.connections import connections
def reset_connections():
c = connections
c._conn = {}
c._kwargs = {}
request.addfinalizer(reset_connections)
client = Mock()
client.search.return_value = dummy_response()
connections.add_connection('mock', client)
return client
def _es_client(worker_id):
"""
Makes the ES test helper client available.
Also patches settings.ES_INDEX_PREFIX using the xdist worker ID so that each process
gets unique indices when running tests using multiple processes using pytest -n.
"""
# pytest's monkeypatch does not work in session fixtures, but there is no need to restore
# the value so we just overwrite it normally
settings.ES_INDEX_PREFIX = f'test_{worker_id}'
from elasticsearch_dsl.connections import connections
client = get_test_client(nowait=False)
connections.add_connection('default', client)
yield client
def client(request):
# inner import to avoid throwing off coverage
from elasticsearch_dsl.connections import connections
# hack to workaround pytest not caching skip on fixtures (#467)
global _client_loaded
if _client_loaded:
skip()
_client_loaded = True
try:
client = get_test_client(nowait='WAIT_FOR_ES' not in os.environ)
connections.add_connection('default', client)
return client
except SkipTest:
skip()
def mock_client(request):
# inner import to avoid throwing off coverage
from elasticsearch_dsl.connections import connections
def reset_connections():
c = connections
c._conn = {}
c._kwargs = {}
request.addfinalizer(reset_connections)
client = Mock()
client.search.return_value = dummy_response()
connections.add_connection('mock', client)
return client
def run(self) -> None:
LOGGER.info('Running search metadata to Elasticsearch task')
try:
# extract records from metadata store
record = self.extractor.extract()
# create connection
connections.add_connection('default', self.elasticsearch_client)
connection = connections.get_connection()
# health check ES
health = connection.cluster.health()
status = health["status"]
if status not in ("green", "yellow"):
msg = f"Elasticsearch healthcheck failed: {status}"
LOGGER.error(msg)
raise Exception(msg)
# create index
LOGGER.info(f"Creating ES index {self.elasticsearch_new_index}")
index = Index(name=self.elasticsearch_new_index, using=self.elasticsearch_client)
index.document(self.document_mapping)
index.create()
# publish search metadata to ES
cnt = 0
for success, info in parallel_bulk(connection,
self.generate_documents(record=record),
raise_on_error=False,
chunk_size=self.elasticsearch_batch_size,
request_timeout=self.elasticsearch_timeout_sec):
if not success:
LOGGER.warn(f"There was an error while indexing a document to ES: {info}")
else:
cnt += 1
if cnt == self.elasticsearch_batch_size:
LOGGER.info(f'Published {str(cnt*self.elasticsearch_batch_size)} records to ES')
# delete old index
self._delete_old_index(connection=connection,
document_index=index)
LOGGER.info("Elasticsearch Indexing completed")
finally:
self._closer.close()
def ready(self):
from elasticsearch import Elasticsearch, RequestsHttpConnection, RequestError, ConnectionTimeout
from elasticsearch_dsl.connections import connections
from requests_aws4auth import AWS4Auth
from django.conf import settings
from shoutit.models import LocationIndex, ShoutIndex
from shoutit.utils import error_logger
import shoutit
# Todo (Nour): Cleanup!
# Define a default global Elasticsearch client
if 'es.amazonaws.com' in settings.ES_URL:
# Connect using IAM based authentication on AWS
awsauth = AWS4Auth(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY, 'eu-west-1',
'es')
ES = Elasticsearch(hosts=[settings.ES_URL],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
connections.add_connection(alias='default', conn=ES)
else:
ES = connections.create_connection(hosts=[settings.ES_URL])
shoutit.ES = ES
# Initiate the index if not initiated
try:
LocationIndex.init()
except RequestError:
pass
except ConnectionTimeout:
error_logger.warn("ES Server is down", exc_info=True)
# Initiate the index if not initiated
try:
ShoutIndex.init()
except RequestError:
pass
except ConnectionTimeout:
error_logger.warn("ES Server is down", exc_info=True)
def get_top_k_scores(text_query):
text_query = ' '.join(pre_process_text(text_query))
es = Elasticsearch()
connections.add_connection('CivilArticle', es)
es_index = 'coliee_bm25_index'
es_query = {
"multi_match": {
"query": text_query,
"fields": ["title", "content"]
}
}
res = es.search(index=es_index,
body={
"from": 0,
"size": k,
"query": es_query
})
res = res['hits']['hits']
count = 0
articles = []
scores = []
i = 0
while i < len(res) and count < k:
try:
if res[i]:
doc_code = res[i]['_source']["code"].lower()
articles.append(doc_code)
scores.append(str(res[i]['_score']))
else:
break
i += 1
except IndexError:
continue
return articles, scores
from datetime import datetime
from elasticsearch import Elasticsearch
from elasticsearch_dsl import Document, Date, Integer, Keyword, Text
from elasticsearch_dsl.connections import connections
HOSTS = ['localhost:9200']
CONNECTION_ALIAS = 'dev'
es_client = Elasticsearch(HOSTS, timeout=5)
connections.add_connection(CONNECTION_ALIAS, es_client)
class Article(Document):
title = Text(analyzer='snowball', fields={'raw': Keyword()})
body = Text(analyzer='snowball')
tags = Keyword()
published_from = Date()
lines = Integer()
class Index:
name = 'blog'
settings = {
"number_of_shards": 2,
}
using = CONNECTION_ALIAS
def save(self, ** kwargs):
self.lines = len(self.body.split())
return super(Article, self).save(** kwargs)
def client(elasticsearch_instance):
"""Return a connection to the elasticsearch server
"""
connection = get_test_client(nowait=True)
connections.add_connection("default", connection)
return connection
def create_app():
global app_created
if not app_created:
BlueprintsManager.register(app)
Migrate(app, db)
app.config.from_object(env('APP_CONFIG',
default='config.ProductionConfig'))
db.init_app(app)
_manager = Manager(app)
_manager.add_command('db', MigrateCommand)
if app.config['CACHING']:
cache.init_app(app, config={'CACHE_TYPE': 'simple'})
else:
cache.init_app(app, config={'CACHE_TYPE': 'null'})
stripe.api_key = 'SomeStripeKey'
app.secret_key = 'super secret key'
app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False
app.config['FILE_SYSTEM_STORAGE_FILE_VIEW'] = 'static'
app.logger.addHandler(logging.StreamHandler(sys.stdout))
app.logger.setLevel(logging.ERROR)
# set up jwt
app.config['JWT_HEADER_TYPE'] = 'JWT'
app.config['JWT_ACCESS_TOKEN_EXPIRES'] = timedelta(days=1)
app.config['JWT_REFRESH_TOKEN_EXPIRES'] = timedelta(days=365)
app.config['JWT_ERROR_MESSAGE_KEY'] = 'error'
app.config['JWT_TOKEN_LOCATION'] = ['cookies', 'headers']
app.config['JWT_REFRESH_COOKIE_PATH'] = '/v1/auth/token/refresh'
app.config['JWT_SESSION_COOKIE'] = False
app.config['JWT_BLACKLIST_ENABLED'] = True
app.config['JWT_BLACKLIST_TOKEN_CHECKS'] = ['refresh']
_jwt = JWTManager(app)
_jwt.user_loader_callback_loader(jwt_user_loader)
_jwt.token_in_blacklist_loader(is_token_blacklisted)
# setup celery
app.config['CELERY_BROKER_URL'] = app.config['REDIS_URL']
app.config['CELERY_RESULT_BACKEND'] = app.config['CELERY_BROKER_URL']
app.config['CELERY_ACCEPT_CONTENT'] = ['json', 'application/text']
CORS(app, resources={r"/*": {"origins": "*"}})
AuthManager.init_login(app)
if app.config['TESTING'] and app.config['PROFILE']:
# Profiling
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])
# development api
with app.app_context():
from app.api.admin_statistics_api.events import event_statistics
from app.api.auth import auth_routes
from app.api.attendees import attendee_misc_routes
from app.api.bootstrap import api_v1
from app.api.celery_tasks import celery_routes
from app.api.event_copy import event_copy
from app.api.exports import export_routes
from app.api.imports import import_routes
from app.api.uploads import upload_routes
from app.api.users import user_misc_routes
from app.api.orders import order_misc_routes
from app.api.role_invites import role_invites_misc_routes
from app.api.auth import ticket_blueprint, authorised_blueprint
from app.api.admin_translations import admin_blueprint
from app.api.orders import alipay_blueprint
from app.api.settings import admin_misc_routes
from app.api.server_version import info_route
app.register_blueprint(api_v1)
app.register_blueprint(event_copy)
app.register_blueprint(upload_routes)
app.register_blueprint(export_routes)
app.register_blueprint(import_routes)
app.register_blueprint(celery_routes)
app.register_blueprint(auth_routes)
app.register_blueprint(event_statistics)
app.register_blueprint(user_misc_routes)
app.register_blueprint(attendee_misc_routes)
app.register_blueprint(order_misc_routes)
app.register_blueprint(role_invites_misc_routes)
app.register_blueprint(ticket_blueprint)
app.register_blueprint(authorised_blueprint)
app.register_blueprint(admin_blueprint)
app.register_blueprint(alipay_blueprint)
app.register_blueprint(admin_misc_routes)
app.register_blueprint(info_route)
add_engine_pidguard(db.engine)
if app.config['SQLALCHEMY_DATABASE_URI'].startswith("sqlite://"):
sqlite_datetime_fix()
sa.orm.configure_mappers()
if app.config['SERVE_STATIC']:
app.add_url_rule('/static/<path:filename>',
endpoint='static',
view_func=app.send_static_file)
# sentry
if not app_created and 'SENTRY_DSN' in app.config:
sentry_sdk.init(app.config['SENTRY_DSN'],
integrations=[
FlaskIntegration(),
RedisIntegration(),
CeleryIntegration(),
SqlalchemyIntegration()
])
# redis
redis_store.init_app(app)
# elasticsearch
if app.config['ENABLE_ELASTICSEARCH']:
client.init_app(app)
connections.add_connection('default', client.elasticsearch)
with app.app_context():
try:
cron_rebuild_events_elasticsearch.delay()
except Exception:
pass
app_created = True
return app, _manager, db, _jwt
def make_app(project_root):
"""
Construct the rubberband app.
This method is being called from server.py, also from bin/rubberband-ctl.
Parameters
----------
project_root : str
Root path of rubberband source code.
Returns
-------
app
the rubberband app
"""
# init logger
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s%(msecs)03d %(levelname)-5s %(name)-15s %(message)s',
datefmt='%d-%m-%Y %H:%M:%S - ')
loggr = logging.getLogger()
loggr.setLevel(level=20)
# Load options from environment
config = "/etc/rubberband/app.cfg"
if os.path.isfile(config):
logging.info(
"Loading additional configuration from /etc/rubberband/app.cfg")
options.parse_config_file(config)
else:
logging.info("Using default config.")
# settings for tornado
settings = {
"debug": True if options.num_processes == 1 else False,
"static_path": os.path.join(project_root, "static"),
"template_path": os.path.join(project_root, "templates"),
"cookie_secret": options.cookie_secret,
"xsrf_cookies": True,
"default_handler_class": ErrorView,
"logger": loggr,
}
# set up tornado application
# From the doc: "A Tornado web application maps URLs or URL patterns to subclasses
# of tornado.web.RequestHandler. Those classes define get() or post() methods
# to handle HTTP GET or POST requests to that URL."
# these patterns are defined in routes.py
# default: app = Application(routes, **settings)
app = Application([(HostMatches(r'(localhost|127\.0\.0\.1|{}|)'.format(
options.prod_url.replace("https://", ""))), routes)], **settings)
logging.info("Setting up Elasticsearch connection.")
# set up elasticsearch
# create connection instance
# the timeout argument is needed when you upload big files
conn = Elasticsearch([options.elasticsearch_url],
use_ssl=options.elasticsearch_use_ssl,
verify_certs=options.elasticsearch_verify_certs,
ca_certs=options.elasticsearch_ca_certs,
timeout=100)
# connect connection to pool
connections.add_connection("default", conn)
# settings of tornado app
if app.settings["debug"]:
app.base_url = "http://127.0.0.1:{}".format(options.port)
else:
app.base_url = options.prod_url
return app
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from django.core.management.base import BaseCommand
from elasticsearch import Elasticsearch
from elasticsearch_dsl.connections import connections
from recipe.models import Recipe
from recipe.search_indexes import RecipeIndex
es = Elasticsearch([
{'host': 'localhost', 'port': 9200},
])
connections.add_connection(alias='default', conn=es)
class Command(BaseCommand):
args = 'No arguments needed'
help = 'Command to index recipe to elasticsearch'
def handle(self, *args, **options):
if not es.indices.exists('recipe'):
RecipeIndex.init()
recipes = Recipe.objects.only(
'id',
'name',
'preparation_time',
'ingredients',
'servings',
'likes',
Result object used to push to ElasticSearch
"""
from datetime import datetime
from elasticsearch_dsl import DocType, String, Date, Integer
from elasticsearch_dsl.connections import connections
from elasticsearch import Elasticsearch, RequestsHttpConnection
"""
Future work: certs, TLS, or any kind of security/auth.
"""
#http_auth=('user', 'secret'),
#verify_certs=True,
#ca_certs='/path/to/cacert.pem',
#client_cert='/path/to/client_cert.pem',
#client_key='/path/to/client_key.pem',
es = Elasticsearch(['localhost'], port=9200, use_ssl=False)
connections.add_connection('default', es)
"""
es_result
Define a result class to be passed to ElasticSearch directly
"""
class es_result(DocType):
source = String()
referrer = String()
data = String()
dataHash = String()
dataBytes = Integer()
regex_hit = Integer()
regex_hits = String()
searchterm_hit = Integer()
def create_app():
global app_created
if not app_created:
BlueprintsManager.register(app)
Migrate(app, db)
app.config.from_object(env('APP_CONFIG', default='config.ProductionConfig'))
db.init_app(app)
_manager = Manager(app)
_manager.add_command('db', MigrateCommand)
if app.config['CACHING']:
cache.init_app(app, config={'CACHE_TYPE': 'simple'})
else:
cache.init_app(app, config={'CACHE_TYPE': 'null'})
stripe.api_key = 'SomeStripeKey'
app.secret_key = 'super secret key'
app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False
app.config['FILE_SYSTEM_STORAGE_FILE_VIEW'] = 'static'
app.logger.addHandler(logging.StreamHandler(sys.stdout))
app.logger.setLevel(logging.ERROR)
# set up jwt
app.config['JWT_AUTH_USERNAME_KEY'] = 'email'
app.config['JWT_EXPIRATION_DELTA'] = timedelta(seconds=24 * 60 * 60)
app.config['JWT_AUTH_URL_RULE'] = '/auth/session'
_jwt = JWT(app, jwt_authenticate, jwt_identity)
# setup celery
app.config['CELERY_BROKER_URL'] = app.config['REDIS_URL']
app.config['CELERY_RESULT_BACKEND'] = app.config['CELERY_BROKER_URL']
CORS(app, resources={r"/*": {"origins": "*"}})
AuthManager.init_login(app)
if app.config['TESTING'] and app.config['PROFILE']:
# Profiling
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])
# development api
with app.app_context():
from app.api.admin_statistics_api.events import event_statistics
from app.api.auth import auth_routes
from app.api.attendees import attendee_misc_routes
from app.api.bootstrap import api_v1
from app.api.celery_tasks import celery_routes
from app.api.event_copy import event_copy
from app.api.exports import export_routes
from app.api.imports import import_routes
from app.api.uploads import upload_routes
from app.api.users import user_misc_routes
from app.api.orders import order_misc_routes
app.register_blueprint(api_v1)
app.register_blueprint(event_copy)
app.register_blueprint(upload_routes)
app.register_blueprint(export_routes)
app.register_blueprint(import_routes)
app.register_blueprint(celery_routes)
app.register_blueprint(auth_routes)
app.register_blueprint(event_statistics)
app.register_blueprint(user_misc_routes)
app.register_blueprint(attendee_misc_routes)
app.register_blueprint(order_misc_routes)
sa.orm.configure_mappers()
if app.config['SERVE_STATIC']:
app.add_url_rule('/static/<path:filename>',
endpoint='static',
view_func=app.send_static_file)
# sentry
if not app_created and 'SENTRY_DSN' in app.config:
sentry.init_app(app, dsn=app.config['SENTRY_DSN'])
# redis
redis_store.init_app(app)
# elasticsearch
if app.config['ENABLE_ELASTICSEARCH']:
client.init_app(app)
connections.add_connection('default', client.elasticsearch)
with app.app_context():
try:
cron_rebuild_events_elasticsearch.delay()
except Exception:
pass
app_created = True
return app, _manager, db, _jwt
def create_app():
global app_created
if not app_created:
BlueprintsManager.register(app)
Migrate(app, db)
app.config.from_object(env('APP_CONFIG', default='config.ProductionConfig'))
db.init_app(app)
_manager = Manager(app)
_manager.add_command('db', MigrateCommand)
if app.config['CACHING']:
cache.init_app(app, config={'CACHE_TYPE': 'simple'})
else:
cache.init_app(app, config={'CACHE_TYPE': 'null'})
stripe.api_key = 'SomeStripeKey'
app.secret_key = 'super secret key'
app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False
app.config['FILE_SYSTEM_STORAGE_FILE_VIEW'] = 'static'
app.logger.addHandler(logging.StreamHandler(sys.stdout))
app.logger.setLevel(logging.ERROR)
# set up jwt
app.config['JWT_AUTH_USERNAME_KEY'] = 'email'
app.config['JWT_EXPIRATION_DELTA'] = timedelta(seconds=24 * 60 * 60)
app.config['JWT_AUTH_URL_RULE'] = '/auth/session'
_jwt = JWT(app, jwt_authenticate, jwt_identity)
# setup celery
app.config['CELERY_BROKER_URL'] = app.config['REDIS_URL']
app.config['CELERY_RESULT_BACKEND'] = app.config['CELERY_BROKER_URL']
app.config['CELERY_ACCEPT_CONTENT'] = ['json', 'application/text']
CORS(app, resources={r"/*": {"origins": "*"}})
AuthManager.init_login(app)
if app.config['TESTING'] and app.config['PROFILE']:
# Profiling
app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])
# development api
with app.app_context():
from app.api.admin_statistics_api.events import event_statistics
from app.api.auth import auth_routes
from app.api.attendees import attendee_misc_routes
from app.api.bootstrap import api_v1
from app.api.celery_tasks import celery_routes
from app.api.event_copy import event_copy
from app.api.exports import export_routes
from app.api.imports import import_routes
from app.api.uploads import upload_routes
from app.api.users import user_misc_routes
from app.api.orders import order_misc_routes
from app.api.role_invites import role_invites_misc_routes
from app.api.auth import ticket_blueprint, authorised_blueprint
from app.api.admin_translations import admin_blueprint
from app.api.orders import alipay_blueprint
from app.api.settings import admin_misc_routes
app.register_blueprint(api_v1)
app.register_blueprint(event_copy)
app.register_blueprint(upload_routes)
app.register_blueprint(export_routes)
app.register_blueprint(import_routes)
app.register_blueprint(celery_routes)
app.register_blueprint(auth_routes)
app.register_blueprint(event_statistics)
app.register_blueprint(user_misc_routes)
app.register_blueprint(attendee_misc_routes)
app.register_blueprint(order_misc_routes)
app.register_blueprint(role_invites_misc_routes)
app.register_blueprint(ticket_blueprint)
app.register_blueprint(authorised_blueprint)
app.register_blueprint(admin_blueprint)
app.register_blueprint(alipay_blueprint)
app.register_blueprint(admin_misc_routes)
sa.orm.configure_mappers()
if app.config['SERVE_STATIC']:
app.add_url_rule('/static/<path:filename>',
endpoint='static',
view_func=app.send_static_file)
# sentry
if not app_created and 'SENTRY_DSN' in app.config:
sentry_sdk.init(app.config['SENTRY_DSN'], integrations=[FlaskIntegration()])
# redis
redis_store.init_app(app)
# elasticsearch
if app.config['ENABLE_ELASTICSEARCH']:
client.init_app(app)
connections.add_connection('default', client.elasticsearch)
with app.app_context():
try:
cron_rebuild_events_elasticsearch.delay()
except Exception:
pass
app_created = True
return app, _manager, db, _jwt
connections.create_connection(hosts=['localhost'], timeout=20)
###############################################
# Multiple clusters
connections.configure(
default={'hosts': 'localhost'},
dev={
'hosts': ['esdev1.example.com:9200'],
'sniff_on_start': True
}
)
# We can also add them later using aliases...
connections.create_connection('qa', hosts=['esqa1.example.com'], sniff_on_start=True)
connections.add_connection('qa', my_client)
# When searching we can refer to a specific connection by using aliases
s = Search(using='qa')
###############################################
# The api is chainable
s = Search().using(client).query('match', title='python')
# Send the request
response = s.execute()
# Requests are cached by default by the python client,
# subsequent calls to execute will not trigger additional
# requests being sent to Elasticsearch
# To force a request specify `ignore_cache=True` when sending
# a request
def es_client():
"""Create and return elasticsearch connection"""
connection = Elasticsearch([os.environ.get('TEST_ES_SERVER', {})])
connections.add_connection('default', connection)
return connection
Phải khai báo lại class Meta với doc_type và tên index đúng như dưới
"""
doc_type = 'CivilArticle'
index = index_name
class Index:
"""
Từ bản 0.10.2 trở đi phải khai báo thêm cả class Index có thuộc tính name là tên index trong elastic search như dưới đây
"""
name = index_name
# Create connection
es = Elasticsearch()
connections.create_connection(hosts=['localhost'], timeout=20)
connections.add_connection('CivilArticle', es)
CivilArticle.init(index_name)
def preprocess_content(content):
res = []
lines = content.split('\n')
for line in lines:
line = remove_numbering(line)
words = pre_process_text(line)
res.append(' '.join(words))
return ' '.join(res)
def insert(doc):
from app.models import AWSKey
from elasticsearch_dsl.connections import connections
from app import app
import elasticsearch_dsl as dsl
try:
es_aws_auth = app.config['ES_AUTH']
es_auth = AWS4Auth(es_aws_auth['key'], es_aws_auth['secret'],
es_aws_auth['region'], 'es')
except:
es_auth = None
client = Elasticsearch(hosts=app.config['ES_HOSTS'],
http_auth=es_auth,
connection_class=RequestsHttpConnection,
sniff_timeout=60)
connections.add_connection('default', client)
SHORT_NAMES = {
"Amazon Elastic Compute Cloud": "EC2",
"Amazon EC2 Container Registry (ECR)": "ECR",
"Amazon Simple Storage Service": "S3",
"Amazon RDS Service": "RDS",
"Amazon ElastiCache": "ElastiCache",
"Amazon Elastic File System": "EFS",
"Amazon Elastic MapReduce": "EMR",
"Amazon Route 53": "Route 53",
"AWS Key Management Service": "KMS",
"Amazon DynamoDB": "DynamoDB",
"Amazon Simple Email Service": "SES",
"Amazon Simple Notification Service": "SNS",
"Amazon Simple Queue Service": "SQS",
from elasticsearch import Elasticsearch, RequestsHttpConnection
"""
Future work: certs, TLS, or any kind of security/auth.
"""
#http_auth=('user', 'secret'),
#verify_certs=True,
#ca_certs='/path/to/cacert.pem',
#client_cert='/path/to/client_cert.pem',
#client_key='/path/to/client_key.pem',
es = Elasticsearch(
['localhost'],
port=9200,
use_ssl=False
)
connections.add_connection('default', es)
"""
es_result
Define a result class to be passed to ElasticSearch directly
"""
class es_result(DocType):
source = String()
referrer = String()
data = String()
dataHash = String()
dataBytes = Integer()
regex_hit = Integer()
regex_hits = String()
searchterm_hit = Integer()
searchterm_hits = String()
