async def update(self, filter_obj: IFilter, to_update: dict) -> int:
"""
Update object in Storage by Query.
:param IFilter filter_obj: filter object which describes what objects need to update
:param dict to_update: dictionary with fields and values which should be updated
:return: number of updated entries
"""
def dict_to_source(__to_update: dict) -> str:
"""
Convert __to_update dict into elasticsearch source representation.
:param __to_update: dictionary with fields and values which should be updated
:return: elasticsearch inline representation
"""
def _value_converter(_value: Any) -> Any:
"""
Convert value if it is necessary
:param _value:
:return:
"""
if isinstance(_value, bool):
return str(_value).lower()
elif isinstance(_value, datetime):
return _value.strftime(self._default_date_format)
return _value
return " ".join((self._inline_templates.get(
type(value), self._default_template).substitute(
FIELD_NAME=key, FIELD_VALUE=_value_converter(value))
for key, value in __to_update.items()))
def _update(_ubq) -> UpdateByQueryResponse:
"""
Perform Update by Query in separate thread.
:param UpdateByQuery _ubq: UpdateByQuery instance
:return: Response object of ElasticSearch DSL module
"""
return _ubq.execute()
_to_update = to_update.copy() # Copy because it will be change below
# NOTE: Important: call of the parent update method changes _to_update dict!
await super().update(filter_obj, _to_update) # Call the generic code
ubq = UpdateByQuery(index=self._index, using=self._es_client)
filter_by = self._query_converter.build(filter_obj)
ubq = ubq.query(filter_by)
source = dict_to_source(_to_update)
ubq = ubq.script(source=source, lang=ESWords.PAINLESS)
result = await self._loop.run_in_executor(self._tread_pool_exec,
_update, ubq)
await self._refresh_index(
) # ensure that updated results will be ready immediately
return result.updated
def test_complex_example():
ubq = UpdateByQuery()
ubq = ubq.query('match', title='python') \
.query(~Q('match', title='ruby')) \
.filter(Q('term', category='meetup') | Q('term', category='conference')) \
.script(source='ctx._source.likes += params.f', lang='painless', params={'f': 3})
ubq.query.minimum_should_match = 2
assert {
'query': {
'bool': {
'filter': [
{
'bool': {
'should': [
{'term': {'category': 'meetup'}},
{'term': {'category': 'conference'}}
]
}
}
],
'must': [ {'match': {'title': 'python'}}],
'must_not': [{'match': {'title': 'ruby'}}],
'minimum_should_match': 2
}
},
'script': {
'source': 'ctx._source.likes += params.f',
'lang': 'painless',
'params': {
'f': 3
}
}
} == ubq.to_dict()
def update_by_query(self, **kwargs):
"""
This function is used to update ElasticSearch entries by record
:param kwargs: provided kwargs
:return:
Example:
>>> update_by_query(fields="publisher", query="oreilly", field_to_update="publisher", new_value="OnMedia")
"""
try:
client = Elasticsearch(
hosts=ELASTIC_HOSTNAME
)
ubq = UpdateByQuery(
using=client,
index=self.book_index
)
search_fields = kwargs["fields"]
query = kwargs["query"]
field_to_update = kwargs["field_to_update"]
new_value = kwargs["new_value"]
update_query = ubq.query(
"multi_match",
query=query,
fields=search_fields
).script(
source="ctx._source.{}='{}'".format(field_to_update, new_value)
)
results = update_query.execute()._d_
return results
except Exception as ex:
print(ex, flush=True)
def on_delete(self, status_id, user_id):
ubq = UpdateByQuery(using=es, index="tweet*")
query = (ubq.query("match", id_str__keyword=str(status_id)).query(
"match", user__id_str__keyword=str(user_id)).script(
source="ctx._source.delete = true"))
resp = query.execute()
print(f"deleted {status_id} {user_id} {resp.to_dict()}")
return
def test_ubq_to_dict():
ubq = UpdateByQuery()
assert {} == ubq.to_dict()
ubq = ubq.query('match', f=42)
assert {"query": {"match": {'f': 42}}} == ubq.to_dict()
assert {"query": {"match": {'f': 42}}, "size": 10} == ubq.to_dict(size=10)
ubq = UpdateByQuery(extra={"size": 5})
assert {"size": 5} == ubq.to_dict()
def test_ubq_to_dict():
ubq = UpdateByQuery()
assert {} == ubq.to_dict()
ubq = ubq.query('match', f=42)
assert {"query": {"match": {'f': 42}}} == ubq.to_dict()
assert {"query": {"match": {'f': 42}}, "size": 10} == ubq.to_dict(size=10)
ubq = UpdateByQuery(extra={"size": 5})
assert {"size": 5} == ubq.to_dict()
def test_complex_example():
ubq = UpdateByQuery()
ubq = (ubq.query("match",
title="python").query(~Q("match", title="ruby")).filter(
Q("term", category="meetup")
| Q("term", category="conference")).script(
source="ctx._source.likes += params.f",
lang="painless",
params={"f": 3}))
ubq.query.minimum_should_match = 2
assert {
"query": {
"bool": {
"filter": [{
"bool": {
"should": [
{
"term": {
"category": "meetup"
}
},
{
"term": {
"category": "conference"
}
},
]
}
}],
"must": [{
"match": {
"title": "python"
}
}],
"must_not": [{
"match": {
"title": "ruby"
}
}],
"minimum_should_match":
2,
}
},
"script": {
"source": "ctx._source.likes += params.f",
"lang": "painless",
"params": {
"f": 3
},
},
} == ubq.to_dict()
def test_ubq_to_dict():
ubq = UpdateByQuery()
assert {} == ubq.to_dict()
ubq = ubq.query("match", f=42)
assert {"query": {"match": {"f": 42}}} == ubq.to_dict()
assert {"query": {"match": {"f": 42}}, "size": 10} == ubq.to_dict(size=10)
ubq = UpdateByQuery(extra={"size": 5})
assert {"size": 5} == ubq.to_dict()
ubq = UpdateByQuery(extra={"extra_q": Q("term", category="conference")})
assert {"extra_q": {"term": {"category": "conference"}}} == ubq.to_dict()
def test_complex_example():
ubq = UpdateByQuery()
ubq = ubq.query('match', title='python') \
.query(~Q('match', title='ruby')) \
.filter(Q('term', category='meetup') | Q('term', category='conference')) \
.script(source='ctx._source.likes += params.f', lang='painless', params={'f': 3})
ubq.query.minimum_should_match = 2
assert {
'query': {
'bool': {
'filter': [{
'bool': {
'should': [{
'term': {
'category': 'meetup'
}
}, {
'term': {
'category': 'conference'
}
}]
}
}],
'must': [{
'match': {
'title': 'python'
}
}],
'must_not': [{
'match': {
'title': 'ruby'
}
}],
'minimum_should_match':
2
}
},
'script': {
'source': 'ctx._source.likes += params.f',
'lang': 'painless',
'params': {
'f': 3
}
}
} == ubq.to_dict()
for user in conf["list"][l]:
id_str = user["id"]
username = user["name"]
follow.append(id_str)
ubq = UpdateByQuery(using=es, index="tweet*")
query = (ubq.query("match", user__id_str__keyword=id_str).script(
source="""
int i = 0;
int found = 0;
if (ctx._source.list_name == null){
ctx._source.list_name = [];
}
for(i = 0; i < ctx._source.list_name.size(); i++){
if (ctx._source.list_name[i] == params.list_name){
found = 1;
break;
}
}
if (found == 0){
ctx._source.list_name.add(params.list_name);
}
ctx._source.user.screen_name = params.username;
""",
params={
"list_name": l,
"username": username
}))
resp = query.execute()
# StreamListenerを継承するクラスListener作成
class Listener(tweepy.StreamListener):
