def test_getitem_missing(self):
provider = ESTermIndexWeightingProvider(self.es, self.index,
initial_weights=[('ba', 1), ('knark', 2), ('knirk', 3), ('foo', 3)])
IndicesClient(self.es).refresh(self.index)
self.assertRaises(KeyError, lambda: provider['notfound'])
self.assertRaises(KeyError, lambda: provider['ba', 'notfound'])
provider = ESTermIndexWeightingProvider(self.es, self.index,
initial_weights=[('ba', 1), ('knark', 2), ('knirk', 3), ('foo', 3)],
missing='ignore')
IndicesClient(self.es).refresh(self.index)
self.assertIsNone(provider['notfound'])
self.assertEqual([('ba', 1)], list(provider['ba', 'notfound']))
def __init__(self,
type='elastic',
url=None,
base=None,
name=None,
server_base=None,
content=None,
default=None):
if not (url and name):
raise Exception('both url and name-parameters required')
self.url = url
self.base = base
self.server_base = server_base
self.index_name = name
self.elastic = Elasticsearch(url)
self.indices = IndicesClient(self.elastic)
self.index_content = content is not None
self.content = content
self.default = default or ''
self.index_map = {}
if not self.indices.exists(self.index_name):
try:
self.indices.create(self.index_name, ignore=400, body=mapping)
self.index_existed = False
except RequestError as e:
if e.error == 'resource_already_exists_exception':
self.indices = IndicesClient(self.elastic)
self.index_existed = True
else:
raise e
else:
self.index_existed = True
for c in (content or {}).get('parts', {}).values():
prefix = content.get('index_prefix', '')
index = prefix + c['index_name']
self.index_map[c['type']] = index
if not self.indices.exists(index):
try:
self.indices.create(index, ignore=400, body=mapping)
except RequestError as e:
if e.error == 'resource_already_exists_exception':
self.indices = IndicesClient(self.elastic)
else:
raise e
def __init__(self, conf, queue):
self.conf = conf
host = self.conf.get("host", "es")
port = self.conf.get("port", 9200)
self.log = logging.getLogger("pulsar.indexer")
logging.getLogger("elasticsearch").setLevel(logging.INFO)
self.log.debug("port: %r" % port)
self.es = Elasticsearch([{"host": host, "port": port}])
self.cluster_client = ClusterClient(self.es)
health = self.cluster_client.health()
if not health or health.get("number_of_nodes") < 1:
raise Exception("No Elasticsearch nodes found: %r" % health)
# Put our template
self.indices_client = IndicesClient(self.es)
self.index_prefix = self.conf.get("index_prefix", self.INDEX_PREFIX)
self.indices_client.put_template(
name=self.index_prefix, body=open("conf/es-template.json").read())
self.log.info("Put template to ES for pulsar indexes")
self.last_event_time = time()
self.index_prefix = self.index_prefix + "-"
self.index_name = self.get_index_name()
self.queue = queue
self.counter = 0
self.stats_checkpoint = time()
self.stats_every = 10000
try:
# This will block as it reads from the queue
self.bulk(self.es, self.iterator(), stats_only=True)
except Exception as e:
self.log.exception("Error with bulk", exc_info=e)
def test_tokenize(self):
test_case = {
"text": "2018-06-08T00:00:00Z INFO GET /v1/bundles/7ef8966b-45ef-4e0a-a51b-44a865372050.2018-06-08T230333.785338Z?param1=1¶m2=2 {\"key\": \"value\"}"
}
index_name = self.es_client._format_today_index_name(self.index_prefix)
index_client = IndicesClient(TestESClient.es)
with self.new_index(index_name):
response = index_client.analyze(index=index_name, body=test_case)
tokens = [t['token'] for t in response['tokens']]
self.assertEqual(set(tokens), {
'7ef8966b-45ef-4e0a-a51b-44a865372050',
'2018-06-08T230333.785338Z',
':',
'INFO',
'1',
'2',
'v1',
'bundles',
'key',
'GET',
'param2',
'param1',
'2018-06-08T00:00:00Z',
'value'
})
self.assertEqual(len(tokens), 14)
def __init__(self, driver, connection_name, connection_config):
super(ElasticsearchConnection, self).__init__(driver, connection_name,
connection_config)
self.uri = self.connection_config.get('uri').split(',')
self.cnx_opts = {}
use_ssl = self.connection_config.get('use_ssl', True)
if isinstance(use_ssl, str):
if use_ssl.lower() == 'false':
use_ssl = False
else:
use_ssl = True
self.cnx_opts['use_ssl'] = use_ssl
if use_ssl:
verify_certs = self.connection_config.get('verify_certs', True)
if isinstance(verify_certs, str):
if verify_certs.lower() == 'false':
verify_certs = False
else:
verify_certs = True
self.cnx_opts['verify_certs'] = verify_certs
self.cnx_opts['ca_certs'] = self.connection_config.get(
'ca_certs', None)
self.cnx_opts['client_cert'] = self.connection_config.get(
'client_cert', None)
self.cnx_opts['client_key'] = self.connection_config.get(
'client_key', None)
self.es = Elasticsearch(self.uri, **self.cnx_opts)
try:
self.log.debug("Elasticsearch info: %s" % self.es.info())
except Exception as e:
self.log.warn("An error occured on estabilishing "
"connection to Elasticsearch: %s" % e)
self.ic = IndicesClient(self.es)
def create_index_conf():
indices_client = IndicesClient(models.client)
index_name = 'conf'
doc_type = index_name
if indices_client.exists(index_name):
indices_client.delete(index=index_name)
indices_client.create(index=index_name)
def test_index_manager_regenerate_indices_from_broken_state(self, *args):
"""
`regenerate_indices` should succeed and give us a working ElasticSearch
when it runs and finds a broken state (eg. with an existing, incorrect
index with the name of an alias).
This can occur when ES restarts and an update signal is triggered before
Richie had a chance to bootstrap ES.
"""
# The indices client will be used to test the actual indices in ElasticSearch
indices_client = IndicesClient(client=ES_CLIENT)
# Create a course and trigger a signal to index it. This will create a
# broken "richie_test_courses" index
course = CourseFactory(should_publish=True)
update_course(course.extended_object, "en")
self.assertIsNotNone(indices_client.get("richie_test_courses"))
# Call our `regenerate_indices command`
creation_datetime = datetime(2010, 1, 1, tzinfo=timezone.utc)
creation_string = creation_datetime.strftime("%Y-%m-%d-%Hh%Mm%S.%fs")
with mock.patch.object(timezone, "now", return_value=creation_datetime):
regenerate_indices(None)
# No error was thrown, the courses index (like all others) was bootstrapped
self.assertIsNotNone(
indices_client.get(f"richie_test_courses_{creation_string}")
)
# The expected alias is associated with the index
self.assertEqual(
list(indices_client.get_alias("richie_test_courses").keys())[0],
f"richie_test_courses_{creation_string}",
)
def __init__(self, host, port, db_config):
self.es = Elasticsearch([{"host": host, "port": port}])
if not self.es.ping():
error(
"Cannot connect to elasticsearch cluster for users. Check database configuration in user_db_config.json."
)
exit(0)
index = db_config["index"]
self.index = index["name"]
user_type = db_config["user_type"]
self.user_type_name = user_type["name"]
mappings = dict()
if "mapping" in user_type:
mappings[self.user_type_name] = user_type["mapping"]
body = dict()
if "settings" in index:
body["settings"] = index["settings"]
if mappings:
body["mappings"] = mappings
try:
self.indices_client = IndicesClient(self.es)
if not self.indices_client.exists(self.index):
self.indices_client.create(index=self.index, body=body)
except TransportError:
error(
"Error while creating elasticsearch cluster for users. Check type mappings in user_db_config.json."
)
print(traceback.format_exc())
exit(0)
def get_es_indices():
es_idx = IndicesClient(es)
indicies = es_idx.get('appcompat-*')
result = []
for index_name,v in indicies.iteritems():
result.append((index_name,index_name,v['settings']['index']['creation_date']))
return result
def handle(self, *args, **options):
es = Elasticsearch(hosts=[{'host': 'localhost', 'port': 9200}])
fop=open('spider/management/commands/'+str(argv[2]), 'r')
inds = IndicesClient(es)
mapping={ "mappings": { "product_type": { "properties": { "code": { "type" : "string" },"name": {"type" : "string"},"img": {"type" : "string"},"url": {"type" : "string"},"price_reg": {"type" : "float"},"price_discount": {"type" : "float"}}}}}
if not inds.exists(index='gearbest_index'):
inds.create(index='gearbest_index',body=mapping)
print 'gearbest_index created'
for jsonline in fop:
jobj=loads(jsonline)
del jobj["_type"]
es.index(index="gearbest_index",doc_type='product_type', body=jobj, id=jobj['code'])
disc=0
reg=0
if len(jobj['price_discount'])>0:
disc = float(jobj['price_discount'][0])
if len(jobj['price_reg'])>0:
reg = float(jobj['price_reg'][0])
#insert="INSERT into 'price_gb' ('price','price_disc','code','date') values ("+str(reg)+", "+str(disc)+", '"+str(jobj['code'])+"', '"+str(datetime.today())+"')"
#cursor = connection.cursor()
#cursor.execute(insert)
add_price=Price_gb(price=reg,price_disc=disc,code=str(jobj['code']),date=datetime.date.today())
add_price.save()
print 'code='+str(jobj['code'])
def _elastic(self, doc_id=None, doc={}, option='create'):
"""
option:
init: 初始化文档结构。（当config.ini中的init为True时才会执行。）
create: 若文档已存在，则不执行任何操作。 若文档不存在，则直接创建。
update: 若文档已存在，则直接更新。 若文档不存在，则不执行任何操作。
delete: 若文档已存在，则直接删除。若文档不存在，则不执行任何操作。
"""
esclient = Elasticsearch([self.elastic])
status = 'Success !'
if 'create' == option:
try:
esclient.create(
index=self.elastic['index'],
doc_type=self.elastic['type'],
id=doc_id,
body=doc,
)
except ConflictError:
status = 'Fail(existsd) !'
elif 'update' == option:
try:
esclient.update(
index=self.elastic['index'],
doc_type=self.elastic['type'],
id=doc_id,
body={'doc': doc},
)
except NotFoundError:
status = 'Fail(not existsd) !'
elif 'delete' == option:
try:
esclient.delete(
index=self.elastic['index'],
doc_type=self.elastic['type'],
id=doc_id,
)
except NotFoundError:
status = 'Fail(not existsd) !'
elif 'init' == option:
try:
IndicesClient(esclient).create(
index=self.elastic['index'],
body=doc,
)
except RequestError:
status = 'Fail(existsd) !'
self.logger.record('Sync@%s < %s-%s-%s > %s' % (
option,
self.elastic['index'],
self.elastic['type'],
doc_id,
status,
))
def _create_index(self):
es_index = IndicesClient(self._es)
if es_index.exists(self._store_index):
logging.info('Index ' + self._store_index +
' already exists. Skipping index creation.')
return None
es_mapping = {
"mappings": {
'last_runtime': {
'properties': {
'plugin_name': {
'index': 'not_analyzed',
'type': 'string'
},
'rule_name': {
'index': 'not_analyzed',
'type': 'string'
},
'plugin_sid': {
'index': 'not_analyzed',
'type': 'long'
},
'@timestamp': {
'format': 'dateOptionalTime||epoch_millis',
'type': 'date'
}
}
}
}
}
self._es.indices.create(self._store_index, body=es_mapping)
time.sleep(1)
def prepare_index(self, courses):
"""
Not a test.
This method is doing the heavy lifting for the tests in this class:
- prepare the Elasticsearch index,
- execute the query.
"""
self.create_filter_pages()
# Index these 4 courses in Elasticsearch
indices_client = IndicesClient(client=ES_CLIENT)
# Delete any existing indices so we get a clean slate
indices_client.delete(index="_all")
# Create an index we'll use to test the ES features
indices_client.create(index="test_courses")
indices_client.close(index="test_courses")
indices_client.put_settings(body=ANALYSIS_SETTINGS,
index="test_courses")
indices_client.open(index="test_courses")
# Use the default courses mapping from the Indexer
indices_client.put_mapping(body=CoursesIndexer.mapping,
doc_type="course",
index="test_courses")
# Add the sorting script
ES_CLIENT.put_script(id="state", body=CoursesIndexer.scripts["state"])
# Actually insert our courses in the index
actions = [{
"_id": course["id"],
"_index": "test_courses",
"_op_type": "create",
"_type": "course",
**course,
} for course in courses]
bulk(actions=actions, chunk_size=500, client=ES_CLIENT)
indices_client.refresh()
def indices(self):
""" Returns an elasticsearch.client.IndicesClient instance """
if not self.connected:
self.connect()
return IndicesClient(self.client)
def create_index_survey():
indices_client = IndicesClient(models.client)
index_name = models.SurveyMap._meta.es_index_name
if indices_client.exists(index_name):
indices_client.delete(index=index_name)
indices_client.create(index=index_name)
#put_settings(models.ScentemotionMap)
# add qstfld fields
es_mapping = models.SurveyMap._meta.es_mapping
for qst, mapping in survey.qst2fld.items():
fields = mapping[0]
field_type = mapping[1]
if field_type == 'nested_qst_ans':
for field in fields:
if field not in es_mapping['properties']:
es_mapping['properties'][field] = {}
es_mapping['properties'][field]['type'] = 'nested'
es_mapping['properties'][field]['properties'] = {}
es_mapping['properties'][field]['properties']['question'] = {'type' : 'text', 'fields' : {'keyword' : {'type' : 'keyword', 'ignore_above' : 256}}}
es_mapping['properties'][field]['properties']['answer'] = {'type' : 'text', 'fields' : {'keyword' : {'type' : 'keyword', 'ignore_above' : 256}}}
#'type' : 'nested',
#'properties' : {
# 'question' : {'type' : 'text', 'fields' : {'keyword' : {'type' : 'keyword', 'ignore_above' : 256}}},
# 'answer' : {'type' : 'text', 'fields' : {'keyword' : {'type' : 'keyword', 'ignore_above' : 256}}},
# }
#},
indices_client.put_mapping(
doc_type=models.SurveyMap._meta.es_type_name,
#body=models.SurveyMap._meta.es_mapping,
body=es_mapping,
index=index_name
)
def setUp(self):
"""
Make sure tests get clean indexes to run
"""
self.indices_client = IndicesClient(client=settings.ES_CLIENT)
# Delete any existing indexes so we get a clean slate
self.indices_client.delete(index="_all")
# Create an index we'll use to test the ES features
self.indices_client.create(index="test_index")
# Add a mapping for a test document type. It needs to include different fields for the
# various features we'll be running tests on
self.indices_client.put_mapping(
body={
"properties": {
"datetime_field": {
"type": "date"
},
"keyword_field": {
"type": "keyword"
},
"text_field": {
"type": "text"
},
}
},
doc_type="test_doc",
index="test_index",
)
def handle(self, *args, **options):
client = IndicesClient(client=settings.ES_CLIENT)
self.recreate_index(
client, **{
'index_name': Sofa._meta.es_index_name,
'doc_type': Sofa._meta.es_type_name,
'body': Sofa._meta.es_mapping
})
self.push_db_to_index(client, SofaSerializerES,
Sofa.objects.all().iterator())
self.recreate_index(
client, **{
'index_name': Bed._meta.es_index_name,
'doc_type': Bed._meta.es_type_name,
'body': Bed._meta.es_mapping
})
self.push_db_to_index(client, BedSerializerES,
Bed.objects.all().iterator())
self.recreate_index(
client, **{
'index_name': Dining._meta.es_index_name,
'doc_type': Dining._meta.es_type_name,
'body': Dining._meta.es_mapping
})
self.push_db_to_index(client, DiningSerializerES,
Dining.objects.all().iterator())
def main():
# Define the globals
global index_names
global STARTED_TIMESTAMP
global es
global es_indices
try:
# Initiate the elasticsearch session using ES low-level client.
# By default nodes are randomized before passed into the pool and round-robin strategy is used for load balancing.
es = Elasticsearch(ES_HOSTS, timeout=30)
es_indices = IndicesClient(es)
except:
print("Could not connect to elasticsearch!")
sys.exit(1)
print("Creating indices.. \n"),
indices = generate_indices()
print("Done!\n")
print("GET Settings \n"),
print json.dumps(es_indices.get_settings(index="_all"),
sort_keys=True,
indent=4,
separators=(',', ': '))
print("Done!\n")
# We will Clean up the indices by default
# Default: True
if CLEANUP:
print("Cleaning up created indices.. "),
cleanup_indices()
print("Done!\n")
def perform_create_index(indexable, logger):
"""
Create a new index in ElasticSearch from an indexable instance
"""
indices_client = IndicesClient(client=settings.ES_CLIENT)
# Create a new index name, suffixing its name with a timestamp
new_index = "{:s}_{:s}".format(
indexable.index_name,
timezone.now().strftime("%Y-%m-%d-%Hh%Mm%S.%fs"))
# Create the new index
logger.info(
'Creating a new Elasticsearch index "{:s}"...'.format(new_index))
indices_client.create(index=new_index)
indices_client.put_mapping(body=indexable.mapping,
doc_type=indexable.document_type,
index=new_index)
# Populate the new index with data provided from our indexable class
bulk(
actions=indexable.get_data_for_es(new_index, "create"),
chunk_size=settings.ES_CHUNK_SIZE,
client=settings.ES_CLIENT,
stats_only=True,
)
# Return the name of the index we just created in ElasticSearch
return new_index
def setUp(self):
"""
Make sure all indexes are deleted before each new test is run.
"""
super().setUp()
self.indices_client = IndicesClient(client=settings.ES_CLIENT)
self.indices_client.delete(index="_all")
def import_examples_into_es(examples: list):
index_name = config.index_name
type_name = config.type_name
buck_size = config.buck_size
es = Elasticsearch(config.es_url)
es_index = IndicesClient(es)
if es_index.exists(index=index_name):
es_index.delete(index=index_name)
# 创建索引
with open(config.es_index_json) as f:
mappings = json.load(f)
res = es.indices.create(index=index_name, body=mappings)
# 数据批量导入es
for i in range(len(examples)):
examples[i] = {
"_index": index_name,
"_type": type_name,
"_id": examples[i]["ntc_id"],
"_source": examples[i]
}
for i in tqdm(range(ceil(len(examples) / buck_size)), desc="Import into ES"):
bulk(es, actions=examples[i * buck_size: min((i + 1) * buck_size, len(examples))])
def setUp(self):
"""
Instantiate our ES client and make sure all indexes are deleted before each test
"""
super().setUp()
self.indices_client = IndicesClient(client=ES_CLIENT)
self.indices_client.delete(index="_all")
def perform_create_index(indexable, logger):
"""
Create a new index in ElasticSearch from an indexable instance
"""
indices_client = IndicesClient(client=ES_CLIENT)
# Create a new index name, suffixing its name with a timestamp
new_index = "{:s}_{:s}".format(
indexable.index_name,
timezone.now().strftime("%Y-%m-%d-%Hh%Mm%S.%fs"))
# Create the new index
logger.info(
'Creating a new Elasticsearch index "{:s}"...'.format(new_index))
indices_client.create(index=new_index)
# The index needs to be closed before we set an analyzer
indices_client.close(index=new_index)
indices_client.put_settings(body=ANALYSIS_SETTINGS, index=new_index)
indices_client.open(index=new_index)
indices_client.put_mapping(body=indexable.mapping,
doc_type=indexable.document_type,
index=new_index)
# Populate the new index with data provided from our indexable class
richie_bulk(indexable.get_es_documents(new_index))
# Return the name of the index we just created in ElasticSearch
return new_index
def reindex(self):
elastic_client = Elasticsearch([{
"host": self.__host,
"port": self.__port
}])
index_client = IndicesClient(elastic_client)
# Create new index with necessory fields mapping
# , master_timeout=10, timeout=10
index_client.create(index=self.__target_index, body=self.__body)
# reindexind data from source index to target index
helpers.reindex(client=elastic_client,
source_index=self.__source_index,
target_index=self.__target_index)
# creating alias for target index
alias = {'actions': []}
# remove_action = {"remove": {"index": self.__source_index, "alias": self.__alias}}
add_action = {
"add": {
"index": self.__target_index,
"alias": self.__alias
}
}
# alias['actions'].append(remove_action)
alias['actions'].append(add_action)
# deleteing the source index
index_client.delete(index=self.__source_index)
index_client.update_aliases(body=alias)
def put_settings(obj):
indices_client = IndicesClient(models.client)
index_name = obj._meta.es_index_name
indices_client.close(index=index_name)
kwargs = {
"analysis": {
"analyzer": {
"default": {
"tokenizer": "standard",
"filter": ["synonym"]
},
"keepwords": {
"tokenizer": "standard",
"filter": ["keepwords"]
},
},
"filter": {
"synonym": {
"type": "synonym",
"synonyms_path": "synonym.txt"
},
"keepwords": {
"type": "keep",
"keep_words_path": "keepwords.txt"
},
}
}
}
indices_client.put_settings(index=index_name, body=kwargs)
indices_client.open(index=index_name)
def setUp(self):
""" Starts a new connector for every test
"""
try:
os.unlink("config.txt")
except OSError:
pass
open("config.txt", "w").close()
self.connector = Connector(
address='%s:%s' % (mongo_host, self.primary_p),
oplog_checkpoint='config.txt',
target_url=elastic_pair,
ns_set=['test.test'],
u_key='_id',
auth_key=None,
doc_manager='mongo_connector/doc_managers/elastic_doc_manager.py',
auto_commit_interval=0)
# Clean out test databases
try:
self.elastic_doc._remove()
except OperationFailed:
try:
# Create test.test index if necessary
client = Elasticsearch(hosts=[elastic_pair])
idx_client = IndicesClient(client)
idx_client.create(index='test.test')
except es_exceptions.TransportError:
pass
self.conn.test.test.drop()
self.connector.start()
assert_soon(lambda: len(self.connector.shard_set) > 0)
assert_soon(lambda: sum(1 for _ in self.elastic_doc._search()) == 0)
def __init__(self, host, port, db_config):
self.es = Elasticsearch([{"host": host, "port": port}])
try:
if self.es.ping():
es_logger = logging.getLogger('elasticsearch')
es_logger.setLevel(logging.CRITICAL)
self.indices_client = IndicesClient(self.es)
index_definitions = db_config["index_definitions"]
self.settings = db_config["settings"]
self.data_point_definition = index_definitions["data_point"]
self.create_index_from_definition(self.data_point_definition,
self.settings)
self.data_point_type_name = self.data_point_definition["name"]
self.data_point_index = self.data_point_definition[
"index_name"]
self.definitions.append(self.data_point_definition)
self.experiment_definition = index_definitions["experiment"]
self.create_index_from_definition(self.experiment_definition,
self.settings)
self.experiment_type_name = self.experiment_definition["name"]
self.experiment_index = self.experiment_definition[
"index_name"]
self.definitions.append(self.experiment_definition)
self.target_system_definition = index_definitions[
"target_system"]
self.create_index_from_definition(
self.target_system_definition, self.settings)
self.target_system_type_name = self.target_system_definition[
"name"]
self.target_system_index = self.target_system_definition[
"index_name"]
self.definitions.append(self.target_system_definition)
self.analysis_definition = index_definitions["analysis"]
self.create_index_from_definition(self.analysis_definition,
self.settings)
self.analysis_type_name = self.analysis_definition["name"]
self.analysis_index = self.analysis_definition["index_name"]
self.definitions.append(self.analysis_definition)
self.stage_definition = index_definitions["stage"]
self.create_index_from_definition(self.stage_definition,
self.settings)
self.stage_type_name = self.stage_definition["name"]
self.stage_index = self.stage_definition["index_name"]
self.definitions.append(self.stage_definition)
else:
raise ConnectionError("Host/port values are not valid")
except TransportError as err1:
error(
"TransportError while creating elasticsearch instance for experiments. Check type mappings in experiment_db_config.json."
)
raise err1
def execute_query(self, querystring=""):
"""
Not a test.
This method is doing the heavy lifting for the tests in this class: create and fill the
index with our courses so we can run our queries and check our facet counts.
It also executes the query and returns the result from the API.
"""
# Create the subject category page. This is necessary to link the subjects we
# defined above with the "subjects" filter
# As it is the only page we create, we expect it to have the path "0001"
CategoryFactory(page_reverse_id="subjects", should_publish=True)
# Index these 4 courses in Elasticsearch
indices_client = IndicesClient(client=ES_CLIENT)
# Delete any existing indices so we get a clean slate
indices_client.delete(index="_all")
# Create an index we'll use to test the ES features
indices_client.create(index="test_courses")
indices_client.close(index="test_courses")
indices_client.put_settings(body=ANALYSIS_SETTINGS, index="test_courses")
indices_client.open(index="test_courses")
# Use the default courses mapping from the Indexer
indices_client.put_mapping(
body=CoursesIndexer.mapping, doc_type="course", index="test_courses"
)
# Add the sorting script
ES_CLIENT.put_script(id="state", body=CoursesIndexer.scripts["state"])
# Actually insert our courses in the index
actions = [
{
"_id": course["id"],
"_index": "test_courses",
"_op_type": "create",
"_type": "course",
"absolute_url": {"en": "url"},
"cover_image": {"en": "image"},
"title": {"en": "title"},
**course,
"course_runs": [
{
"languages": course_run["languages"],
"start": arrow.utcnow().datetime,
"end": arrow.utcnow().datetime,
"enrollment_start": arrow.utcnow().datetime,
"enrollment_end": arrow.utcnow().datetime,
}
for course_run in course["course_runs"]
],
}
for course in COURSES
]
bulk(actions=actions, chunk_size=500, client=ES_CLIENT)
indices_client.refresh()
response = self.client.get(f"/api/v1.0/courses/?{querystring:s}")
self.assertEqual(response.status_code, 200)
return json.loads(response.content)
def execute_query(self, courses, querystring="", **extra):
"""
Not a test.
Prepare the ElasticSearch index and execute the query in it.
"""
indices_client = IndicesClient(client=ES_CLIENT)
# Delete any existing indices so we get a clean slate
indices_client.delete(index="_all")
# Create an index we'll use to test the ES features
indices_client.create(index=COURSES_INDEX)
# The index needs to be closed before we set an analyzer
indices_client.close(index=COURSES_INDEX)
indices_client.put_settings(body=ANALYSIS_SETTINGS,
index=COURSES_INDEX)
indices_client.open(index=COURSES_INDEX)
# Use the default courses mapping from the Indexer
indices_client.put_mapping(body=CoursesIndexer.mapping,
doc_type="course",
index=COURSES_INDEX)
# Add the sorting script
ES_CLIENT.put_script(id="score", body=CoursesIndexer.scripts["score"])
ES_CLIENT.put_script(id="state_field",
body=CoursesIndexer.scripts["state_field"])
# Actually insert our courses in the index
actions = [{
"_id": course["id"],
"_index": COURSES_INDEX,
"_op_type": "create",
"_type": "course",
"absolute_url": {
"en": "en/url",
"fr": "fr/url"
},
"categories": ["1", "2", "3"],
"cover_image": {
"en": "en/image",
"fr": "fr/image"
},
"is_meta": False,
"logo": {
"en": "/en/some/img.png",
"fr": "/fr/some/img.png"
},
"nb_children": 0,
"organizations": ["11", "12", "13"],
**course,
} for course in courses]
bulk(actions=actions, chunk_size=500, client=ES_CLIENT)
indices_client.refresh()
results = self.client.get(
f"/api/v1.0/courses/autocomplete/?{querystring:s}", **extra)
self.assertEqual(results.status_code, 200)
return json.loads(results.content)
def recreate_index(self, index_name, index_mapping):
indices_client = IndicesClient(client=ES_CLIENT)
if indices_client.exists(index_name):
indices_client.delete(index=index_name)
indices_client.create(index=index_name)
indices_client.put_mapping(doc_type='page',
index=index_name,
body=index_mapping)
