async def update(self, filter_obj: IFilter, to_update: dict) -> int: """ Update object in Storage by Query. :param IFilter filter_obj: filter object which describes what objects need to update :param dict to_update: dictionary with fields and values which should be updated :return: number of updated entries """ def dict_to_source(__to_update: dict) -> str: """ Convert __to_update dict into elasticsearch source representation. :param __to_update: dictionary with fields and values which should be updated :return: elasticsearch inline representation """ def _value_converter(_value: Any) -> Any: """ Convert value if it is necessary :param _value: :return: """ if isinstance(_value, bool): return str(_value).lower() elif isinstance(_value, datetime): return _value.strftime(self._default_date_format) return _value return " ".join((self._inline_templates.get( type(value), self._default_template).substitute( FIELD_NAME=key, FIELD_VALUE=_value_converter(value)) for key, value in __to_update.items())) def _update(_ubq) -> UpdateByQueryResponse: """ Perform Update by Query in separate thread. :param UpdateByQuery _ubq: UpdateByQuery instance :return: Response object of ElasticSearch DSL module """ return _ubq.execute() _to_update = to_update.copy() # Copy because it will be change below # NOTE: Important: call of the parent update method changes _to_update dict! await super().update(filter_obj, _to_update) # Call the generic code ubq = UpdateByQuery(index=self._index, using=self._es_client) filter_by = self._query_converter.build(filter_obj) ubq = ubq.query(filter_by) source = dict_to_source(_to_update) ubq = ubq.script(source=source, lang=ESWords.PAINLESS) result = await self._loop.run_in_executor(self._tread_pool_exec, _update, ubq) await self._refresh_index( ) # ensure that updated results will be ready immediately return result.updated
def test_complex_example(): ubq = UpdateByQuery() ubq = ubq.query('match', title='python') \ .query(~Q('match', title='ruby')) \ .filter(Q('term', category='meetup') | Q('term', category='conference')) \ .script(source='ctx._source.likes += params.f', lang='painless', params={'f': 3}) ubq.query.minimum_should_match = 2 assert { 'query': { 'bool': { 'filter': [ { 'bool': { 'should': [ {'term': {'category': 'meetup'}}, {'term': {'category': 'conference'}} ] } } ], 'must': [ {'match': {'title': 'python'}}], 'must_not': [{'match': {'title': 'ruby'}}], 'minimum_should_match': 2 } }, 'script': { 'source': 'ctx._source.likes += params.f', 'lang': 'painless', 'params': { 'f': 3 } } } == ubq.to_dict()
def update_by_query(self, **kwargs): """ This function is used to update ElasticSearch entries by record :param kwargs: provided kwargs :return: Example: >>> update_by_query(fields="publisher", query="oreilly", field_to_update="publisher", new_value="OnMedia") """ try: client = Elasticsearch( hosts=ELASTIC_HOSTNAME ) ubq = UpdateByQuery( using=client, index=self.book_index ) search_fields = kwargs["fields"] query = kwargs["query"] field_to_update = kwargs["field_to_update"] new_value = kwargs["new_value"] update_query = ubq.query( "multi_match", query=query, fields=search_fields ).script( source="ctx._source.{}='{}'".format(field_to_update, new_value) ) results = update_query.execute()._d_ return results except Exception as ex: print(ex, flush=True)
def on_delete(self, status_id, user_id): ubq = UpdateByQuery(using=es, index="tweet*") query = (ubq.query("match", id_str__keyword=str(status_id)).query( "match", user__id_str__keyword=str(user_id)).script( source="ctx._source.delete = true")) resp = query.execute() print(f"deleted {status_id} {user_id} {resp.to_dict()}") return
def test_ubq_to_dict(): ubq = UpdateByQuery() assert {} == ubq.to_dict() ubq = ubq.query('match', f=42) assert {"query": {"match": {'f': 42}}} == ubq.to_dict() assert {"query": {"match": {'f': 42}}, "size": 10} == ubq.to_dict(size=10) ubq = UpdateByQuery(extra={"size": 5}) assert {"size": 5} == ubq.to_dict()
def test_complex_example(): ubq = UpdateByQuery() ubq = (ubq.query("match", title="python").query(~Q("match", title="ruby")).filter( Q("term", category="meetup") | Q("term", category="conference")).script( source="ctx._source.likes += params.f", lang="painless", params={"f": 3})) ubq.query.minimum_should_match = 2 assert { "query": { "bool": { "filter": [{ "bool": { "should": [ { "term": { "category": "meetup" } }, { "term": { "category": "conference" } }, ] } }], "must": [{ "match": { "title": "python" } }], "must_not": [{ "match": { "title": "ruby" } }], "minimum_should_match": 2, } }, "script": { "source": "ctx._source.likes += params.f", "lang": "painless", "params": { "f": 3 }, }, } == ubq.to_dict()
def test_ubq_to_dict(): ubq = UpdateByQuery() assert {} == ubq.to_dict() ubq = ubq.query("match", f=42) assert {"query": {"match": {"f": 42}}} == ubq.to_dict() assert {"query": {"match": {"f": 42}}, "size": 10} == ubq.to_dict(size=10) ubq = UpdateByQuery(extra={"size": 5}) assert {"size": 5} == ubq.to_dict() ubq = UpdateByQuery(extra={"extra_q": Q("term", category="conference")}) assert {"extra_q": {"term": {"category": "conference"}}} == ubq.to_dict()
def test_complex_example(): ubq = UpdateByQuery() ubq = ubq.query('match', title='python') \ .query(~Q('match', title='ruby')) \ .filter(Q('term', category='meetup') | Q('term', category='conference')) \ .script(source='ctx._source.likes += params.f', lang='painless', params={'f': 3}) ubq.query.minimum_should_match = 2 assert { 'query': { 'bool': { 'filter': [{ 'bool': { 'should': [{ 'term': { 'category': 'meetup' } }, { 'term': { 'category': 'conference' } }] } }], 'must': [{ 'match': { 'title': 'python' } }], 'must_not': [{ 'match': { 'title': 'ruby' } }], 'minimum_should_match': 2 } }, 'script': { 'source': 'ctx._source.likes += params.f', 'lang': 'painless', 'params': { 'f': 3 } } } == ubq.to_dict()
for user in conf["list"][l]: id_str = user["id"] username = user["name"] follow.append(id_str) ubq = UpdateByQuery(using=es, index="tweet*") query = (ubq.query("match", user__id_str__keyword=id_str).script( source=""" int i = 0; int found = 0; if (ctx._source.list_name == null){ ctx._source.list_name = []; } for(i = 0; i < ctx._source.list_name.size(); i++){ if (ctx._source.list_name[i] == params.list_name){ found = 1; break; } } if (found == 0){ ctx._source.list_name.add(params.list_name); } ctx._source.user.screen_name = params.username; """, params={ "list_name": l, "username": username })) resp = query.execute() # StreamListenerを継承するクラスListener作成 class Listener(tweepy.StreamListener):