def get_result_query(self, **query):
try:
from base_app.models.elasticsearch.base_model import ElasticSearchModel
e = ElasticSearchModel(_id=str(query['item_id']),
body=query['body'],
index=query['index'],
doc_type=query['doc_type'])
result = {}
if query['function'] == 'search':
result = e.search()
elif query['function'] == 'insert':
result = e.insert()
elif query['function'] == 'delete':
result = e.delete()
elif query['function'] == 'update':
result = e.update()
elif query['function'] == 'count_all':
result = e.count_all()
elif query['function'] == 'count':
result = e.count()
elif query['function'] == 'get_one':
result = e.get_one()
self.result['value'] = result
self.result['status'] = True
return self.result
except:
Debug.get_exception(sub_system='admin',
severity='error',
tags='mongodb > get_all',
data='collection > elastic_statistic')
return self.result
def get_result_query(self, **query):
try:
from base_app.models.elasticsearch.base_model import ElasticSearchModel
e = ElasticSearchModel(_id=str(query['item_id']), body=query['body'], index=query['index'], doc_type=query['doc_type'])
result = {}
if query['function'] == 'search':
result = e.search()
elif query['function'] == 'insert':
result = e.insert()
elif query['function'] == 'delete':
result = e.delete()
elif query['function'] == 'update':
result = e.update()
elif query['function'] == 'count_all':
result = e.count_all()
elif query['function'] == 'count':
result = e.count()
elif query['function'] == 'get_one':
result = e.get_one()
self.result['value'] = result
self.result['status'] = True
return self.result
except:
Debug.get_exception(sub_system='admin', severity='error', tags='mongodb > get_all', data='collection > elastic_statistic')
return self.result
def is_exist(self):
try:
body = {
"filter": {
"or": {
"filters": [{
"and": {
"filters": [{
"query": {
"term": {
"hash_title":
self.get_hash(self.title)
}
}
}, {
"query": {
"term": {
"agency": self.agency,
}
}
}]
}
}, {
"query": {
"term": {
"hash_link": self.get_hash(self.link)
}
}
}]
}
}
}
e = ElasticSearchModel(doc_type=BriefsModel.doc_type,
body=body).search()
if e['hits']['total']:
_id = e['hits']['hits'][0]['_id']
if self.content == str(ContentModel().titr1):
_body = {
"script": "ctx._source.content = __content",
"params": {
"__content": str(ContentModel().titr1)
}
}
ElasticSearchModel(doc_type=BriefsModel.doc_type,
body=_body,
_id=_id).update()
return _id
return False
except:
return True
def insert(self):
try:
body = {
'link': self.link,
'hash_link': self.get_hash(self.link),
'title': self.title,
'hash_title': self.get_hash(self.title),
'ro_title': self.ro_title,
'summary': self.summary,
'thumbnail': self.thumbnail,
'agency': self.agency,
'subject': self.subject,
'content': self.content,
'date': datetime.datetime.today()
}
e = self.is_exist()
if e is False:
self.result['value'] = ElasticSearchModel(
doc_type=BriefsModel.doc_type, body=body).insert()
self.result['status'] = True
self.result['message'] = 'INSERT'
else:
self.result['status'] = False
self.result['message'] = 'EXIST'
self.result['value'] = {'_id': e}
return self.result
except:
Debug.get_exception(sub_system='engine_feed',
severity='critical_error',
tags='insert_brief',
data=self.link)
return self.result
def get_all_backup(self, _page=0, _size=100):
try:
body = {
"from": _page * _size,
"size": _size,
"query": {
"match_all": {}
},
"sort": {
"date": {
"order": "desc"
}
}
}
r = ElasticSearchModel(doc_type=BriefsModel.doc_type,
body=body).search()
for b in r['hits']['hits']:
self.value.append(dict(_id=b['_id'], _source=b['_source']))
self.result['value'] = self.value
self.result['status'] = True
return self.result
except:
Debug.get_exception(sub_system='admin',
severity='error',
tags='briefs > get_all',
data='doc_type: ' + BriefsModel.doc_type)
return self.result
def insert(self):
t = Timer()
d = datetime.datetime.now()
body = {
'link': self.link,
'hash_link': Hash.hash(self.link),
'title': self.title,
'hash_title': Hash.hash(self.title),
'ro_title': self.ro_title,
'summary': self.summary,
'body': self.body,
'thumbnail': self.thumbnail,
'agency': self.agency,
'subject': self.subject,
'category': self.category,
'direction': self.direction,
'date': self.date,
'content': self.content,
'image': self.image,
'images': self.images,
'video': self.video,
'sound': self.sound,
'geographic': self.geographic,
'group': self.group,
'read_date': d,
'read_timestamp': int(time.mktime(d.timetuple())),
}
news = CreateId().create_object_id()
self.result['value'] = ElasticSearchModel(doc_type=self.doc_type, body=body,
_id=news).insert()
self.insert_mongodb(body)
self.result['status'] = True
self.time_insert = t.end()
return self.result
def restore(self, body):
try:
body = {
'link': body['_source']['link'],
'hash_link': body['_source']['hash_link'],
'title': body['_source']['title'],
'hash_title': body['_source']['hash_title'],
'ro_title': body['_source']['ro_title'],
'summary': body['_source']['summary'],
'thumbnail': body['_source']['thumbnail'],
'agency': body['_source']['agency'],
'subject': body['_source']['subject'],
'content': body['_source']['content'],
'date': body['_source']['date']
}
self.result['value'] = ElasticSearchModel(
doc_type=BriefsModel.doc_type, body=body,
_id=body['_id']).insert()
self.result['status'] = True
self.result['message'] = 'INSERT'
return self.result
except:
Debug.get_exception(sub_system='engine_feed',
severity='critical_error',
tags='insert_brief',
data=self.link)
return self.result
def get_top_elements(self, _key, _size):
try:
key_word_query = KeyWordClass(user_keyword=self.user_keyword).get_query_keyword()
body = {
"size": 0,
"query": {
"filtered": {
"filter": {
"and": {
"filters": [
{
"range": {
"date": {"lt": self.end.isoformat(), "gte": self.start.isoformat()}
}
}
] + key_word_query
}
},
}
},
"aggregations": {
"group_by": {
"terms": {"field": _key, "size": _size}
}
}
}
r = ElasticSearchModel(doc_type=self.doc_type, body=body).search()
result = []
for b in r['aggregations']['group_by']['buckets']:
result.append(dict(key=b['key'], doc_count=b['doc_count']))
return result
except:
return []
def is_exist(self):
try:
t = Timer()
if self.title is not None:
body = {
"query": {
"filtered": {
"filter": {
"or": {
"filters": [
{
"and": {
"filters": [
{
"query": {
"term": {
"hash_title": Hash.hash(self.title)
}
}
},
{
"query": {
"term": {
"agency": self.agency,
}
}
}
]
}
},
{
"query": {
"term": {
"hash_link": Hash.hash(self.link)
}
}
}
]
}
}
}
}
}
else:
body = {
"query": {
"term": {
"hash_link": Hash.hash(self.link)
}
}
}
x = ElasticSearchModel(doc_type=self.doc_type, body=body).count()
self.time_is_exist = t.end()
if x:
return True
return False
except:
return False
def get_count_all():
try:
r = ElasticSearchModel(doc_type=BriefsModel.doc_type).count_all()
if r:
return r
return 0
except:
Debug.get_exception(sub_system='admin',
severity='error',
tags='briefs > get_all',
data='doc_type: ' + BriefsModel.doc_type)
return 0
def get_all_log(self, start=None, end=None):
try:
body = {
"query": {
"range": {
"read_date": {
"lt": end.isoformat(),
"gte": start.isoformat()
}
}
}
}
r = ElasticSearchModel(doc_type=NewsLogChartsModel.doc_type, body=body).count()
body = {
"size": r,
"filter": {
"range": {
"read_date": {
"lt": end.isoformat(),
"gte": start.isoformat()
}
}
}
}
r = ElasticSearchModel(doc_type=NewsLogChartsModel.doc_type, body=body).search()
for b in r['hits']['hits']:
self.get_news_log_charts(b['_source'])
self.result['value'] = dict(all_news=self.all_news, category=self.category,
agency=self.agency, subject=self.subject, group=self.group)
self.result['status'] = True
return self.result
except:
Debug.get_exception(send=False)
return self.result
def get_one(self):
try:
r = ElasticSearchModel(doc_type=BriefsModel.doc_type,
_id=self.id).get_one()
self.get_brief(r['_source'], r['_id'])
self.result['value'] = self.value[0]
self.result['status'] = True
return self.result
except:
Debug.get_exception(sub_system='admin',
severity='error',
tags='briefs > count_all',
data='doc_type: ' + BriefsModel.doc_type)
return self.result
def get_all(self):
try:
body = {"from": 0, "size": 1000000, "query": {"match_all": {}}}
r = ElasticSearchModel(doc_type=BriefsModel.doc_type,
body=body).search()
for b in r['hits']['hits']:
self.get_brief(b['_source'], b['_id'])
self.result['value'] = self.value
self.result['status'] = True
return self.result
except:
Debug.get_exception(sub_system='admin',
severity='error',
tags='briefs > count_all',
data='doc_type: ' + BriefsModel.doc_type)
return self.result
def get_top_categories(self, _size_category, _size_content):
try:
key_word_query = KeyWordClass(user_keyword=self.user_keyword).get_query_keyword()
body = {
"size": 0,
"query": {
"filtered": {
"filter": {
"and": {
"filters": [
{
"range": {
"date": {"lt": self.end.isoformat(), "gte": self.start.isoformat()}
}
}
] + key_word_query
}
},
}
},
"aggregations": {
"category": {
"terms": {"field": "category", "size": _size_category},
"aggregations": {
"content": {
"terms": {"field": "content", "size": _size_content}
}
}
}
}
}
r = ElasticSearchModel(doc_type=self.doc_type, body=body).search()
result = []
for b in r['aggregations']['category']['buckets']:
contents = []
for c in b['content']['buckets']:
contents.append(dict(key=c['key'], doc_count=c['doc_count']))
result.append(dict(key=b['key'], doc_count=b['doc_count'], contents=contents))
return result
except:
return []
def get_top_directions(self, _size_direction, _size_agency):
try:
key_word_query = KeyWordClass(user_keyword=self.user_keyword).get_query_keyword()
body = {
"size": 0,
"query": {
"filtered": {
"filter": {
"and": {
"filters": [
{
"range": {
"date": {"lt": self.end.isoformat(), "gte": self.start.isoformat()}
}
}
] + key_word_query
}
},
}
},
"aggregations": {
"direction": {
"terms": {"field": "direction", "size": _size_direction},
"aggregations": {
"agency": {
"terms": {"field": "agency", "size": _size_agency}
}
}
}
}
}
r = ElasticSearchModel(doc_type=self.doc_type, body=body).search()
result = []
for b in r['aggregations']['direction']['buckets']:
agencies = []
for c in b['agency']['buckets']:
agencies.append(dict(key=c['key'], doc_count=c['doc_count']))
result.append(dict(key=b['key'], doc_count=b['doc_count'], agencies=agencies))
return result
except:
return []
def update_news_hash_title(self, __title):
try:
body = {
"script":
"ctx._source.hash_title = __read_date;ctx._source.content = __content",
"params": {
"__read_date": self.get_hash(__title),
"__content": str(ContentModel().news)
}
}
return ElasticSearchModel(doc_type=BriefsModel.doc_type,
body=body,
_id=self.id).update()
except:
Debug.get_exception(sub_system='admin',
severity='error',
tags='briefs > get_all',
data='doc_type: ' + BriefsModel.doc_type)
return self.result
def update_subject_briefs(self):
try:
body = {
"script": "ctx._source.subject = __read_date",
"params": {
"__read_date": "5637944446b9a0342e9bb253"
}
}
r = ElasticSearchModel(doc_type=BriefsModel.doc_type,
body=body,
_id=self.id).update()
self.result['value'] = r
self.result['status'] = True
return self.result
except:
Debug.get_exception(sub_system='admin',
severity='error',
tags='briefs > get_all',
data='doc_type: ' + BriefsModel.doc_type)
return self.result
def get_top_dates(self):
try:
key_word_query = KeyWordClass(user_keyword=self.user_keyword).get_query_keyword()
body = {
"size": 0,
"query": {
"filtered": {
"filter": {
"and": {
"filters": [
{
"range": {
"date": {"lt": self.end.isoformat(), "gte": self.start.isoformat()}
}
}
] + key_word_query
}
},
}
},
"aggregations": {
"date": {
"date_histogram": {
"field": "date",
"interval": "1d"
}
}
}
}
r = ElasticSearchModel(doc_type=self.doc_type, body=body).search()
result = []
for b in r['aggregations']['date']['buckets']:
date = khayyam.JalaliDatetime().fromtimestamp(b['key'] / 1e3).date()
result.append(dict(key=str(date), doc_count=b['doc_count']))
result.reverse()
return result
except:
return []
def update(self):
t = Timer()
body = {
"script": "ctx._source.ro_title = __ro_title;ctx._source.image = __image;"
"ctx._source.body = __body;ctx._source.video = __video;"
"ctx._source.sound = __sound;ctx._source.images = __images",
"params": {
"__ro_title": self.ro_title,
"__image": self.image,
"__body": self.body,
"__video": self.video,
"__sound": self.sound,
"__images": self.images,
}
}
if self.date is not None:
body['script'] += ";ctx._source.date = __date"
body['params']['__date'] = self.date
self.result['value'] = ElasticSearchModel(doc_type=self.doc_type, body=body,
_id=self.id).update()
self.result['status'] = True
self.time_update = t.end()
return self.result
def get_chart_content_direction(self, direction):
def get_direction(__news):
for _i in direction:
if _i['news'] == __news:
return _i['direction']
return False
def get_count_direction(__agency, __dir):
a = 0
for _i in direction:
if _i['agency'] == __agency and _i['direction'] == __dir:
a += 1
return a
try:
directions = DirectionModel().get_all('content')['value']
key_word_query = KeyWordClass(user_keyword=self.user_keyword).get_query_keyword()
direction_count = dict(no_direction=0)
contents = []
series = [dict(id='no_direction', name='بدون جهت گیری', data=[])]
for _d in directions:
series.append(dict(id=str(_d['id']), name=_d['name'], data=[]))
contents.append(dict(id=str(_d['id']), title=_d['name'], value=0))
direction_count[str(_d['id'])] = 0
body = {
"size": 1000000,
"fields": ["_id", "agency"],
"query": {
"filtered": {
"filter": {
"and": {
"filters": [{
"range": {
"date": {
"lt": self.end.isoformat(),
"gte": self.start.isoformat()
}
}
}] + key_word_query
}
}
}
}
}
news = ElasticSearchModel(doc_type=self.doc_type, body=body).search()
try:
count_all = news['hits']['total']
except:
count_all = 0
for n in news['hits']['hits']:
__dir = get_direction(n['_id'])
if __dir is not False:
direction_count[str(__dir)] += 1
else:
direction_count['no_direction'] += 1
for s in contents:
try:
s['value'] = direction_count[s['id']]
except:
s['value'] = 0
categories = []
count_agency = AgencyModel().count_all()['value']
__agencies = self.get_top_elements("agency", count_agency)
for ag in __agencies:
direction_count = dict(no_direction=0)
for _d in directions:
direction_count[str(_d['id'])] = 0
agency = AgencyModel(_id=ObjectId(ag['key'])).get_one()
_c = 0
for _dir in directions:
_x = get_count_direction(ObjectId(ag['key']), _dir['id'])
direction_count[str(_dir['id'])] = _x
_c += _x
series[0]['data'].append(ag['doc_count'] - _c)
for s in series[1:]:
try:
s['data'].append(direction_count[str(s['id'])])
except:
s['data'].append(0)
categories.append(agency['name'])
self.result['value'] = dict(contents=contents, series=series, categories=categories, count_all=count_all)
self.result['status'] = True
return self.result
except:
Debug.get_exception(send=False)
return self.result
def delete(self):
try:
return ElasticSearchModel(doc_type=BriefsModel.doc_type,
_id=self.id).delete()
except:
return False
def get_important_keyword_news(self, direction):
def get_direction(__news):
for _i in direction:
if _i['news'] == __news:
return _i['direction']
return False
try:
keyword_queries = KeyWordClass(user_keyword=self.user_keyword).get_list_query_keyword()
directions = DirectionModel().get_all('content')['value']
series = []
for _d in directions:
series.append(dict(id=str(_d['id']), name=_d['name'], data=[]))
series.append(dict(id='no_direction', name='بدون جهت گیری', data=[]))
categories = []
keywords = []
for query in keyword_queries:
direction_count = dict(no_direction=0)
for _d in directions:
direction_count[str(_d['id'])] = 0
body = {
"size": 1000000,
"fields": ["_id"],
"query": {
"filtered": {
"filter": {
"and": {
"filters": [{
"range": {
"date": {
"lt": self.end.isoformat(),
"gte": self.start.isoformat()
}
}
}] + [query['query']]
}
}
}
}
}
news = ElasticSearchModel(doc_type=self.doc_type, body=body).search()
try:
count_all = news['hits']['total']
except:
count_all = 0
for n in news['hits']['hits']:
__dir = get_direction(n['_id'])
if __dir is not False:
direction_count[str(__dir)] += 1
else:
direction_count['no_direction'] += 1
for s in series:
try:
s['data'].append(direction_count[s['id']])
except:
s['data'].append(0)
keywords.append(dict(keyword=query['keyword'], count=count_all))
categories.append(query['keyword'])
keywords = sorted(keywords, key=lambda k: k['count'], reverse=False)[:3]
self.result['value'] = dict(categories=categories, series=series, keywords=keywords)
self.result['status'] = True
return self.result
except:
Debug.get_exception(send=False)
return self.result