def get_result_query(self, **query): try: from base_app.models.elasticsearch.base_model import ElasticSearchModel e = ElasticSearchModel(_id=str(query['item_id']), body=query['body'], index=query['index'], doc_type=query['doc_type']) result = {} if query['function'] == 'search': result = e.search() elif query['function'] == 'insert': result = e.insert() elif query['function'] == 'delete': result = e.delete() elif query['function'] == 'update': result = e.update() elif query['function'] == 'count_all': result = e.count_all() elif query['function'] == 'count': result = e.count() elif query['function'] == 'get_one': result = e.get_one() self.result['value'] = result self.result['status'] = True return self.result except: Debug.get_exception(sub_system='admin', severity='error', tags='mongodb > get_all', data='collection > elastic_statistic') return self.result
def get_result_query(self, **query): try: from base_app.models.elasticsearch.base_model import ElasticSearchModel e = ElasticSearchModel(_id=str(query['item_id']), body=query['body'], index=query['index'], doc_type=query['doc_type']) result = {} if query['function'] == 'search': result = e.search() elif query['function'] == 'insert': result = e.insert() elif query['function'] == 'delete': result = e.delete() elif query['function'] == 'update': result = e.update() elif query['function'] == 'count_all': result = e.count_all() elif query['function'] == 'count': result = e.count() elif query['function'] == 'get_one': result = e.get_one() self.result['value'] = result self.result['status'] = True return self.result except: Debug.get_exception(sub_system='admin', severity='error', tags='mongodb > get_all', data='collection > elastic_statistic') return self.result
def is_exist(self): try: body = { "filter": { "or": { "filters": [{ "and": { "filters": [{ "query": { "term": { "hash_title": self.get_hash(self.title) } } }, { "query": { "term": { "agency": self.agency, } } }] } }, { "query": { "term": { "hash_link": self.get_hash(self.link) } } }] } } } e = ElasticSearchModel(doc_type=BriefsModel.doc_type, body=body).search() if e['hits']['total']: _id = e['hits']['hits'][0]['_id'] if self.content == str(ContentModel().titr1): _body = { "script": "ctx._source.content = __content", "params": { "__content": str(ContentModel().titr1) } } ElasticSearchModel(doc_type=BriefsModel.doc_type, body=_body, _id=_id).update() return _id return False except: return True
def insert(self): try: body = { 'link': self.link, 'hash_link': self.get_hash(self.link), 'title': self.title, 'hash_title': self.get_hash(self.title), 'ro_title': self.ro_title, 'summary': self.summary, 'thumbnail': self.thumbnail, 'agency': self.agency, 'subject': self.subject, 'content': self.content, 'date': datetime.datetime.today() } e = self.is_exist() if e is False: self.result['value'] = ElasticSearchModel( doc_type=BriefsModel.doc_type, body=body).insert() self.result['status'] = True self.result['message'] = 'INSERT' else: self.result['status'] = False self.result['message'] = 'EXIST' self.result['value'] = {'_id': e} return self.result except: Debug.get_exception(sub_system='engine_feed', severity='critical_error', tags='insert_brief', data=self.link) return self.result
def get_all_backup(self, _page=0, _size=100): try: body = { "from": _page * _size, "size": _size, "query": { "match_all": {} }, "sort": { "date": { "order": "desc" } } } r = ElasticSearchModel(doc_type=BriefsModel.doc_type, body=body).search() for b in r['hits']['hits']: self.value.append(dict(_id=b['_id'], _source=b['_source'])) self.result['value'] = self.value self.result['status'] = True return self.result except: Debug.get_exception(sub_system='admin', severity='error', tags='briefs > get_all', data='doc_type: ' + BriefsModel.doc_type) return self.result
def insert(self): t = Timer() d = datetime.datetime.now() body = { 'link': self.link, 'hash_link': Hash.hash(self.link), 'title': self.title, 'hash_title': Hash.hash(self.title), 'ro_title': self.ro_title, 'summary': self.summary, 'body': self.body, 'thumbnail': self.thumbnail, 'agency': self.agency, 'subject': self.subject, 'category': self.category, 'direction': self.direction, 'date': self.date, 'content': self.content, 'image': self.image, 'images': self.images, 'video': self.video, 'sound': self.sound, 'geographic': self.geographic, 'group': self.group, 'read_date': d, 'read_timestamp': int(time.mktime(d.timetuple())), } news = CreateId().create_object_id() self.result['value'] = ElasticSearchModel(doc_type=self.doc_type, body=body, _id=news).insert() self.insert_mongodb(body) self.result['status'] = True self.time_insert = t.end() return self.result
def restore(self, body): try: body = { 'link': body['_source']['link'], 'hash_link': body['_source']['hash_link'], 'title': body['_source']['title'], 'hash_title': body['_source']['hash_title'], 'ro_title': body['_source']['ro_title'], 'summary': body['_source']['summary'], 'thumbnail': body['_source']['thumbnail'], 'agency': body['_source']['agency'], 'subject': body['_source']['subject'], 'content': body['_source']['content'], 'date': body['_source']['date'] } self.result['value'] = ElasticSearchModel( doc_type=BriefsModel.doc_type, body=body, _id=body['_id']).insert() self.result['status'] = True self.result['message'] = 'INSERT' return self.result except: Debug.get_exception(sub_system='engine_feed', severity='critical_error', tags='insert_brief', data=self.link) return self.result
def get_top_elements(self, _key, _size): try: key_word_query = KeyWordClass(user_keyword=self.user_keyword).get_query_keyword() body = { "size": 0, "query": { "filtered": { "filter": { "and": { "filters": [ { "range": { "date": {"lt": self.end.isoformat(), "gte": self.start.isoformat()} } } ] + key_word_query } }, } }, "aggregations": { "group_by": { "terms": {"field": _key, "size": _size} } } } r = ElasticSearchModel(doc_type=self.doc_type, body=body).search() result = [] for b in r['aggregations']['group_by']['buckets']: result.append(dict(key=b['key'], doc_count=b['doc_count'])) return result except: return []
def is_exist(self): try: t = Timer() if self.title is not None: body = { "query": { "filtered": { "filter": { "or": { "filters": [ { "and": { "filters": [ { "query": { "term": { "hash_title": Hash.hash(self.title) } } }, { "query": { "term": { "agency": self.agency, } } } ] } }, { "query": { "term": { "hash_link": Hash.hash(self.link) } } } ] } } } } } else: body = { "query": { "term": { "hash_link": Hash.hash(self.link) } } } x = ElasticSearchModel(doc_type=self.doc_type, body=body).count() self.time_is_exist = t.end() if x: return True return False except: return False
def get_count_all(): try: r = ElasticSearchModel(doc_type=BriefsModel.doc_type).count_all() if r: return r return 0 except: Debug.get_exception(sub_system='admin', severity='error', tags='briefs > get_all', data='doc_type: ' + BriefsModel.doc_type) return 0
def get_all_log(self, start=None, end=None): try: body = { "query": { "range": { "read_date": { "lt": end.isoformat(), "gte": start.isoformat() } } } } r = ElasticSearchModel(doc_type=NewsLogChartsModel.doc_type, body=body).count() body = { "size": r, "filter": { "range": { "read_date": { "lt": end.isoformat(), "gte": start.isoformat() } } } } r = ElasticSearchModel(doc_type=NewsLogChartsModel.doc_type, body=body).search() for b in r['hits']['hits']: self.get_news_log_charts(b['_source']) self.result['value'] = dict(all_news=self.all_news, category=self.category, agency=self.agency, subject=self.subject, group=self.group) self.result['status'] = True return self.result except: Debug.get_exception(send=False) return self.result
def get_one(self): try: r = ElasticSearchModel(doc_type=BriefsModel.doc_type, _id=self.id).get_one() self.get_brief(r['_source'], r['_id']) self.result['value'] = self.value[0] self.result['status'] = True return self.result except: Debug.get_exception(sub_system='admin', severity='error', tags='briefs > count_all', data='doc_type: ' + BriefsModel.doc_type) return self.result
def get_all(self): try: body = {"from": 0, "size": 1000000, "query": {"match_all": {}}} r = ElasticSearchModel(doc_type=BriefsModel.doc_type, body=body).search() for b in r['hits']['hits']: self.get_brief(b['_source'], b['_id']) self.result['value'] = self.value self.result['status'] = True return self.result except: Debug.get_exception(sub_system='admin', severity='error', tags='briefs > count_all', data='doc_type: ' + BriefsModel.doc_type) return self.result
def get_top_categories(self, _size_category, _size_content): try: key_word_query = KeyWordClass(user_keyword=self.user_keyword).get_query_keyword() body = { "size": 0, "query": { "filtered": { "filter": { "and": { "filters": [ { "range": { "date": {"lt": self.end.isoformat(), "gte": self.start.isoformat()} } } ] + key_word_query } }, } }, "aggregations": { "category": { "terms": {"field": "category", "size": _size_category}, "aggregations": { "content": { "terms": {"field": "content", "size": _size_content} } } } } } r = ElasticSearchModel(doc_type=self.doc_type, body=body).search() result = [] for b in r['aggregations']['category']['buckets']: contents = [] for c in b['content']['buckets']: contents.append(dict(key=c['key'], doc_count=c['doc_count'])) result.append(dict(key=b['key'], doc_count=b['doc_count'], contents=contents)) return result except: return []
def get_top_directions(self, _size_direction, _size_agency): try: key_word_query = KeyWordClass(user_keyword=self.user_keyword).get_query_keyword() body = { "size": 0, "query": { "filtered": { "filter": { "and": { "filters": [ { "range": { "date": {"lt": self.end.isoformat(), "gte": self.start.isoformat()} } } ] + key_word_query } }, } }, "aggregations": { "direction": { "terms": {"field": "direction", "size": _size_direction}, "aggregations": { "agency": { "terms": {"field": "agency", "size": _size_agency} } } } } } r = ElasticSearchModel(doc_type=self.doc_type, body=body).search() result = [] for b in r['aggregations']['direction']['buckets']: agencies = [] for c in b['agency']['buckets']: agencies.append(dict(key=c['key'], doc_count=c['doc_count'])) result.append(dict(key=b['key'], doc_count=b['doc_count'], agencies=agencies)) return result except: return []
def update_news_hash_title(self, __title): try: body = { "script": "ctx._source.hash_title = __read_date;ctx._source.content = __content", "params": { "__read_date": self.get_hash(__title), "__content": str(ContentModel().news) } } return ElasticSearchModel(doc_type=BriefsModel.doc_type, body=body, _id=self.id).update() except: Debug.get_exception(sub_system='admin', severity='error', tags='briefs > get_all', data='doc_type: ' + BriefsModel.doc_type) return self.result
def update_subject_briefs(self): try: body = { "script": "ctx._source.subject = __read_date", "params": { "__read_date": "5637944446b9a0342e9bb253" } } r = ElasticSearchModel(doc_type=BriefsModel.doc_type, body=body, _id=self.id).update() self.result['value'] = r self.result['status'] = True return self.result except: Debug.get_exception(sub_system='admin', severity='error', tags='briefs > get_all', data='doc_type: ' + BriefsModel.doc_type) return self.result
def get_top_dates(self): try: key_word_query = KeyWordClass(user_keyword=self.user_keyword).get_query_keyword() body = { "size": 0, "query": { "filtered": { "filter": { "and": { "filters": [ { "range": { "date": {"lt": self.end.isoformat(), "gte": self.start.isoformat()} } } ] + key_word_query } }, } }, "aggregations": { "date": { "date_histogram": { "field": "date", "interval": "1d" } } } } r = ElasticSearchModel(doc_type=self.doc_type, body=body).search() result = [] for b in r['aggregations']['date']['buckets']: date = khayyam.JalaliDatetime().fromtimestamp(b['key'] / 1e3).date() result.append(dict(key=str(date), doc_count=b['doc_count'])) result.reverse() return result except: return []
def update(self): t = Timer() body = { "script": "ctx._source.ro_title = __ro_title;ctx._source.image = __image;" "ctx._source.body = __body;ctx._source.video = __video;" "ctx._source.sound = __sound;ctx._source.images = __images", "params": { "__ro_title": self.ro_title, "__image": self.image, "__body": self.body, "__video": self.video, "__sound": self.sound, "__images": self.images, } } if self.date is not None: body['script'] += ";ctx._source.date = __date" body['params']['__date'] = self.date self.result['value'] = ElasticSearchModel(doc_type=self.doc_type, body=body, _id=self.id).update() self.result['status'] = True self.time_update = t.end() return self.result
def get_chart_content_direction(self, direction): def get_direction(__news): for _i in direction: if _i['news'] == __news: return _i['direction'] return False def get_count_direction(__agency, __dir): a = 0 for _i in direction: if _i['agency'] == __agency and _i['direction'] == __dir: a += 1 return a try: directions = DirectionModel().get_all('content')['value'] key_word_query = KeyWordClass(user_keyword=self.user_keyword).get_query_keyword() direction_count = dict(no_direction=0) contents = [] series = [dict(id='no_direction', name='بدون جهت گیری', data=[])] for _d in directions: series.append(dict(id=str(_d['id']), name=_d['name'], data=[])) contents.append(dict(id=str(_d['id']), title=_d['name'], value=0)) direction_count[str(_d['id'])] = 0 body = { "size": 1000000, "fields": ["_id", "agency"], "query": { "filtered": { "filter": { "and": { "filters": [{ "range": { "date": { "lt": self.end.isoformat(), "gte": self.start.isoformat() } } }] + key_word_query } } } } } news = ElasticSearchModel(doc_type=self.doc_type, body=body).search() try: count_all = news['hits']['total'] except: count_all = 0 for n in news['hits']['hits']: __dir = get_direction(n['_id']) if __dir is not False: direction_count[str(__dir)] += 1 else: direction_count['no_direction'] += 1 for s in contents: try: s['value'] = direction_count[s['id']] except: s['value'] = 0 categories = [] count_agency = AgencyModel().count_all()['value'] __agencies = self.get_top_elements("agency", count_agency) for ag in __agencies: direction_count = dict(no_direction=0) for _d in directions: direction_count[str(_d['id'])] = 0 agency = AgencyModel(_id=ObjectId(ag['key'])).get_one() _c = 0 for _dir in directions: _x = get_count_direction(ObjectId(ag['key']), _dir['id']) direction_count[str(_dir['id'])] = _x _c += _x series[0]['data'].append(ag['doc_count'] - _c) for s in series[1:]: try: s['data'].append(direction_count[str(s['id'])]) except: s['data'].append(0) categories.append(agency['name']) self.result['value'] = dict(contents=contents, series=series, categories=categories, count_all=count_all) self.result['status'] = True return self.result except: Debug.get_exception(send=False) return self.result
def delete(self): try: return ElasticSearchModel(doc_type=BriefsModel.doc_type, _id=self.id).delete() except: return False
def get_important_keyword_news(self, direction): def get_direction(__news): for _i in direction: if _i['news'] == __news: return _i['direction'] return False try: keyword_queries = KeyWordClass(user_keyword=self.user_keyword).get_list_query_keyword() directions = DirectionModel().get_all('content')['value'] series = [] for _d in directions: series.append(dict(id=str(_d['id']), name=_d['name'], data=[])) series.append(dict(id='no_direction', name='بدون جهت گیری', data=[])) categories = [] keywords = [] for query in keyword_queries: direction_count = dict(no_direction=0) for _d in directions: direction_count[str(_d['id'])] = 0 body = { "size": 1000000, "fields": ["_id"], "query": { "filtered": { "filter": { "and": { "filters": [{ "range": { "date": { "lt": self.end.isoformat(), "gte": self.start.isoformat() } } }] + [query['query']] } } } } } news = ElasticSearchModel(doc_type=self.doc_type, body=body).search() try: count_all = news['hits']['total'] except: count_all = 0 for n in news['hits']['hits']: __dir = get_direction(n['_id']) if __dir is not False: direction_count[str(__dir)] += 1 else: direction_count['no_direction'] += 1 for s in series: try: s['data'].append(direction_count[s['id']]) except: s['data'].append(0) keywords.append(dict(keyword=query['keyword'], count=count_all)) categories.append(query['keyword']) keywords = sorted(keywords, key=lambda k: k['count'], reverse=False)[:3] self.result['value'] = dict(categories=categories, series=series, keywords=keywords) self.result['status'] = True return self.result except: Debug.get_exception(send=False) return self.result