def index_objects(self, index, objects_iterable, bulk_size, coll_name=None):
coll_name = coll_name or self.default_coll_name
if not coll_name:
raise ValueError("No coll_name given")
doctype = index.name
keys = set(index.fields)
keys.update(("id", "_id"))
n = 0
if not bulk_size or bulk_size <= 0:
for obj in objects_iterable:
obj = object_to_dict(obj, keys=keys)
obj_id = obj.pop("_id", None) or obj.pop("id", None)
if obj_id is not None:
self.session.put("/%s/%s/%s" % (coll_name, doctype, obj_id), data=obj)
else:
self.session.post("/%s/%s/" % (coll_name, doctype), data=obj)
n += 1
else:
for obj_batch in batch_iterable(objects_iterable, bulk_size):
batch = []
for obj in obj_batch:
obj = object_to_dict(obj, keys=keys)
obj_id = obj.pop("_id", None) or obj.pop("id", None)
if obj_id is not None:
batch.append(({"index": {"_id": obj_id}}, obj))
else:
batch.append(({"index": {}}, obj))
resp = self.session.bulk("post", "/%s/%s/_bulk" % (coll_name, doctype), data=batch).json()
n += len(resp.get("items") or ())
return n
def to_dict(self):
res = {}
if self.filter:
query = FilteredQuery(self.query, self.filter)
else:
query = self.query
res["query"] = object_to_dict(query)
if self.fields is not None:
res['fields'] = self.fields
if self.sort:
res['sort'] = self.sort
if self.highlight:
if isinstance(self.highlight, (tuple, list, set)):
highlight = HighlightSpec(fields=dict(
(field, HighlightFieldSpec()) for field in self.highlight))
else:
highlight = self.highlight
res["highlight"] = object_to_dict(highlight)
return res
def get_mapping(self):
mapping_json = {}
mapping_json["_source"] = {"enabled": self.enable_source}
if self.enable_size:
mapping_json["_size"] = {
"enabled": True,
"store": True,
"type": "int"
}
if self.enable_size:
mapping_json["_timestamp"] = {
"enabled": True,
"store": True,
"type": "date"
}
properties = mapping_json["properties"] = {}
for field_name, field in self.fields.items():
if field_name == "_all":
mapping_json["_all"] = object_to_dict(field)
else:
assert isinstance(field, SearchField)
properties[field_name] = object_to_dict(field)
return mapping_json
def as_dict(self):
return {
"filtered": {
"query": object_to_dict(self.query),
"filter": object_to_dict(self.filter)
}
}
def to_dict(self):
res = {}
if self.filter:
query = FilteredQuery(self.query, self.filter)
else:
query = self.query
res["query"] = object_to_dict(query)
if self.fields is not None:
res['fields'] = self.fields
if self.sort:
res['sort'] = self.sort
if self.highlight:
if isinstance(self.highlight, (tuple, list, set)):
highlight = HighlightSpec(fields=dict(
(field, HighlightFieldSpec()) for field in self.highlight))
else:
highlight = self.highlight
res["highlight"] = object_to_dict(highlight)
return res
def as_dict(self):
out = {
"must": [object_to_dict(d) for d in self.must] if self.must else None,
"must_not": [object_to_dict(d) for d in self.must_not] if self.must_not else None,
"should": [object_to_dict(d) for d in self.should] if self.should else None,
"boost": self.boost,
"minimum_should_match": self.minimum_should_match
}
return {"bool": _clean_dict(out)}
def search(self, search, indexes=None, count=50,
start=0, page=None, coll_name=None):
coll_name = coll_name or self.default_coll_name
if isinstance(
search, string_types): # This is a silly default, I suppose
search = Search(QueryStringQuery(search))
if isinstance(search, Query):
search = Search(query=search)
search_obj = object_to_dict(search)
if indexes:
url = "/%s/%s/_search" % (coll_name,
",".join(i.name for i in indexes))
else:
url = "/%s/_search" % coll_name
search_obj["from"] = int(start)
search_obj["size"] = int(count)
if page is not None:
search_obj["from"] = search_obj["size"] * page
data = self.session.get(url, data=search_obj).json()
return SearchResults(search=search, raw_result=data,
start=start, count=count)
def search(self,
search,
indexes=None,
count=50,
start=0,
page=None,
coll_name=None):
coll_name = coll_name or self.default_coll_name
if isinstance(search,
string_types): # This is a silly default, I suppose
search = Search(QueryStringQuery(search))
if isinstance(search, Query):
search = Search(query=search)
search_obj = object_to_dict(search)
if indexes:
url = "/%s/%s/_search" % (coll_name, ",".join(i.name
for i in indexes))
else:
url = "/%s/_search" % coll_name
search_obj["from"] = int(start)
search_obj["size"] = int(count)
if page is not None:
search_obj["from"] = search_obj["size"] * page
data = self.session.get(url, data=search_obj).json()
return SearchResults(search=search,
raw_result=data,
start=start,
count=count)
def get_mapping(self):
mapping_json = {}
mapping_json["_source"] = {"enabled": self.enable_source}
if self.enable_size:
mapping_json["_size"] = {"enabled": True, "store": True, "type": "int"}
if self.enable_size:
mapping_json["_timestamp"] = {"enabled": True, "store": True, "type": "date"}
properties = mapping_json["properties"] = {}
for field_name, field in self.fields.items():
if field_name == "_all":
mapping_json["_all"] = object_to_dict(field)
else:
assert isinstance(field, SearchField)
properties[field_name] = object_to_dict(field)
return mapping_json
def index_objects(self,
index,
objects_iterable,
bulk_size,
coll_name=None):
coll_name = coll_name or self.default_coll_name
if not coll_name:
raise ValueError("No coll_name given")
doctype = index.name
keys = set(index.fields)
keys.update(("id", "_id"))
n = 0
if not bulk_size or bulk_size <= 0:
for obj in objects_iterable:
obj = object_to_dict(obj, keys=keys)
obj_id = obj.pop("_id", None) or obj.pop("id", None)
if obj_id is not None:
self.session.put("/%s/%s/%s" %
(coll_name, doctype, obj_id),
data=obj)
else:
self.session.post("/%s/%s/" % (coll_name, doctype),
data=obj)
n += 1
else:
for obj_batch in batch_iterable(objects_iterable, bulk_size):
batch = []
for obj in obj_batch:
obj = object_to_dict(obj, keys=keys)
obj_id = obj.pop("_id", None) or obj.pop("id", None)
if obj_id is not None:
batch.append(({"index": {"_id": obj_id}}, obj))
else:
batch.append(({"index": {}}, obj))
resp = self.session.bulk("post",
"/%s/%s/_bulk" % (coll_name, doctype),
data=batch).json()
n += len(resp.get("items") or ())
return n
def to_dict(self):
out = super(HighlightSpec, self).to_dict()
out["fields"] = dict((name, object_to_dict(spec))
for (name, spec) in self.fields.items())
return out
def as_dict(self):
return {
self._kind: {
"filters": [object_to_dict(f) for f in self.filters]
}
}
def to_dict(self):
out = super(HighlightSpec, self).to_dict()
out["fields"] = dict((name, object_to_dict(spec))
for (name, spec) in self.fields.items())
return out
def as_dict(self):
return {
"constant_score": {
"filter": object_to_dict(self.filter)
}
}