def identify_exist_detect_task_count(submit_user):
query_body = {
'query': {
'bool': {
'must': [{
'term': {
'submit_user': submit_user
}
}, {
'range': {
'detect_process': {
'lt': 99
}
}
}, {
'term': {
'task_type': 'detect'
}
}]
}
}
}
try:
task_exist_result = es_group_result.search(
index=group_index_name, doc_type=group_index_type,
body=query_body)['hits']['hits']
except:
task_exist_result = []
task_exist_count = len(task_exist_result)
return task_exist_count
def identify_exist_analysis_task_count(submit_user):
query_body = {
'query': {
'bool': {
'must': [{
'term': {
'submit_user': submit_user
}
}, {
'term': {
'task_type': 'analysis'
}
}, {
'term': {
'status': 0
}
}]
}
}
}
try:
task_exist_result = es_group_result.search(
index=group_index_name, doc_type=group_index_type,
body=query_body)['hits']['hits']
except:
task_exist_result = []
task_exist_count = len(task_exist_result)
return task_exist_count
def get_group_detect(submit_user):
results = []
query_body = {
'query':{
'filtered':{
'filter':{
'bool':{
'must':[
{'term': {'submit_user': submit_user}},
{'term': {'task_type': 'detect'}}
]
}
}
}
},
'sort':[{'submit_date': {'order': 'desc'}}],
'size': MAX_VALUE
}
#search group task
try:
group_task_result = es_group_result.search(index=group_index_name, doc_type=group_index_type,\
body=query_body)['hits']['hits']
except:
group_task_result = []
#group task results
for group_item in group_task_result:
source = group_item['_source']
task_name = source['task_name']
task_process = source['detect_process']
submit_ts = source['submit_date']
submit_date = ts2date(submit_ts)
state = source['state']
task_type = source['detect_type']
results.append([task_name, submit_date, state, task_type, task_process])
return results
def submit_task(input_data):
print 'aaaaaaaaaaaaaaa'
print input_data
status = 0 # mark it can not submit
task_name = input_data['task_name']
submit_user = input_data['submit_user']
task_id = submit_user + '-' + task_name
#identify the compute task is not more than limit
try:
task_max_count = input_data['task_max_count']
except:
task_max_count = 0
query_body = {
'query':{
'filtered':{
'filter':{
'bool':{
'must':[
{'term': {'submit_user': submit_user}},
{'term': {'status': 0}}
]
}
}
}
}
}
print es_group_result,group_index_name,group_index_type
exist_compute_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits']
exist_compute_count = len(exist_compute_result)
if exist_compute_count >= task_max_count:
return 'more than limit'
#identify the task name is valid
try:
result = es_group_result.get(index=group_index_name, doc_type=group_index_type, id=task_id)['_source']
except:
status = 1
if status != 0 and 'uid_file' not in input_data:
input_data['status'] = 0 # mark the task not compute
count = len(input_data['uid_list'])
input_data['count'] = count
input_data['task_type'] = 'analysis'
input_data['submit_user'] = submit_user
input_data['detect_type'] = ''
input_data['detect_process'] = ''
input_data['task_id'] = task_id
add_es_dict = {'task_information': input_data, 'query_condition':''}
# print es_group_result,group_index_name,group_index_type
# print r
# print '**********************8'
es_group_result.index(index=group_index_name, doc_type=group_index_type, id=task_id, body=input_data)
#print es_group_result,group_index_name,group_index_type
r.lpush(group_analysis_queue_name, json.dumps(input_data))
#print status
print status
return status
def search_detect_task(task_name, submit_date, state, process, detect_type, submit_user):
results = []
query = [{'match':{'task_type': 'detect'}}]
condition_num = 0
if task_name:
task_name_list = task_name.split(' ')
for item in task_name_list:
query.append({'wildcard':{'task_name': '*'+item+'*'}})
condition_num += 1
if submit_date:
submit_date_ts = datetime2ts(submit_date)
submit_date_from = submit_date_ts
submit_date_to = submit_date_ts + DAY
query.append({'range':{'submit_date':{'gte':submit_date_from, 'lt':submit_date_to}}})
condition_num += 1
if state:
state_list = state.split(' ')
for item in state_list:
query.append({'wildcard':{'state': '*'+item+'*'}})
condition_num += 1
if process:
query.append({'range':{'detect_process':{'from': int(process), 'to': MAX_PROCESS}}})
condition_num += 1
if detect_type:
detect_type_list = detect_type.split(',')
nest_body_list = []
for type_item in detect_type_list:
nest_body_list.append({'wildcard':{'detect_type': '*'+type_item+'*'}})
query.append({'bool':{'should': nest_body_list}})
condition_num += 1
if submit_user:
query.append({'term':{'submit_user': submit_user}})
condition_num += 1
try:
search_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
body={'query':{'bool': {'must': query}}, 'sort':[{'submit_date': {'order': 'desc'}}], 'size':MAX_VALUE})['hits']['hits']
except:
search_result = []
#get group information table
for group_item in search_result:
source = group_item['_source']
task_name = source['task_name']
submit_date = ts2datetime(int(source['submit_date']))
submit_user = source['submit_user']
detect_type = source['detect_type']
state = source['state']
process = source['detect_process']
results.append([task_name, submit_user, submit_date, detect_type, state, process])
return results
def search_detect_task(task_name, submit_date, state, process, detect_type, submit_user):
results = []
query = [{'match':{'task_type': 'detect'}}]
condition_num = 0
if task_name:
task_name_list = task_name.split(' ')
for item in task_name_list:
query.append({'wildcard':{'task_name': '*'+item+'*'}})
condition_num += 1
if submit_date:
submit_date_ts = datetime2ts(submit_date)
submit_date_from = submit_date_ts
submit_date_to = submit_date_ts + DAY
query.append({'range':{'submit_date':{'gte':submit_date_from, 'lt':submit_date_to}}})
condition_num += 1
if state:
state_list = state.split(' ')
for item in state_list:
query.append({'wildcard':{'state': '*'+item+'*'}})
condition_num += 1
if process:
query.append({'range':{'detect_process':{'from': int(process), 'to': MAX_PROCESS}}})
condition_num += 1
if detect_type:
detect_type_list = detect_type.split(',')
nest_body_list = []
for type_item in detect_type_list:
nest_body_list.append({'wildcard':{'detect_type': '*'+type_item+'*'}})
query.append({'bool':{'should': nest_body_list}})
condition_num += 1
if submit_user:
query.append({'wildcard':{'submit_user': '******'+submit_user+'*'}})
condition_num += 1
try:
search_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
body={'query':{'bool': {'must': query}}, 'sort':[{'submit_date': {'order': 'desc'}}], 'size':MAX_VALUE})['hits']['hits']
except:
search_result = []
#get group information table
for group_item in search_result:
source = group_item['_source']
task_name = source['task_name']
submit_date = ts2datetime(int(source['submit_date']))
submit_user = source['submit_user']
detect_type = source['detect_type']
state = source['state']
process = source['detect_process']
results.append([task_name, submit_user, submit_date, detect_type, state, process])
return results
def get_group_analysis(submit_user):
results = []
#step1: get query body
query_body = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'submit_user': submit_user
}
}, {
'term': {
'task_type': 'analysis'
}
}]
}
}
}
},
'sort': [{
'submit_date': {
'order': 'desc'
}
}],
'size': MAX_VALUE
}
#step2: search
try:
group_task_result = es_group_result.search(index=group_index_name, doc_type=group_index_type,\
body=query_body)['hits']['hits']
except:
group_task_result = []
#step3: task results
for group_item in group_task_result:
source = group_item['_source']
task_name = source['task_name']
if not task_name:
continue
task_status = source['status']
submit_ts = source['submit_date']
submit_date = ts2date(submit_ts)
try:
state = source['state']
except:
state = ''
results.append([task_name, submit_date, state, task_status])
return results
def submit_task(input_data):
status = 0 # mark it can not submit
task_name = input_data['task_name']
submit_user = input_data['submit_user']
task_id = submit_user + '-' + task_name
#identify the compute task is not more than limit
try:
task_max_count = input_data['task_max_count']
except:
task_max_count = 0
query_body = {
'query':{
'filtered':{
'filter':{
'bool':{
'must':[
{'term': {'submit_user': submit_user}},
{'term': {'status': 0}}
]
}
}
}
}
}
exist_compute_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits']
exist_compute_count = len(exist_compute_result)
if exist_compute_count >= task_max_count:
return 'more than limit'
#identify the task name is valid
try:
result = es_group_result.get(index=group_index_name, doc_type=group_index_type, id=task_id)['_source']
except:
status = 1
if status != 0 and 'uid_file' not in input_data:
input_data['status'] = 0 # mark the task not compute
count = len(input_data['uid_list'])
input_data['count'] = count
input_data['task_type'] = 'analysis'
input_data['submit_user'] = submit_user
input_data['detect_type'] = ''
input_data['detect_process'] = ''
add_es_dict = {'task_information': input_data, 'query_condition':''}
es_group_result.index(index=group_index_name, doc_type=group_index_type, id=task_id, body=input_data)
r.lpush(group_analysis_queue_name, json.dumps(input_data))
return status
def get_group_detect(submit_user):
results = []
query_body = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'submit_user': submit_user
}
}, {
'term': {
'task_type': 'detect'
}
}]
}
}
}
},
'sort': [{
'submit_date': {
'order': 'desc'
}
}],
'size': MAX_VALUE
}
#search group task
try:
group_task_result = es_group_result.search(index=group_index_name, doc_type=group_index_type,\
body=query_body)['hits']['hits']
except:
group_task_result = []
#group task results
for group_item in group_task_result:
source = group_item['_source']
task_name = source['task_name']
task_process = source['detect_process']
submit_ts = source['submit_date']
submit_date = ts2date(submit_ts)
state = source['state']
task_type = source['detect_type']
results.append(
[task_name, submit_date, state, task_type, task_process])
return results
def identify_exist_analysis_task_count(submit_user):
query_body = {
'query':{
'bool':{
'must':[
{'term':{'submit_user': submit_user}},
{'term': {'task_type': 'analysis'}},
{'term': {'status': 0}}
]
}
}
}
try:
task_exist_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits']
except:
task_exist_result = []
task_exist_count = len(task_exist_result)
return task_exist_count
def identify_exist_detect_task_count(submit_user):
query_body = {
'query':{
'bool':{
'must':[
{'term': {'submit_user':submit_user}},
{'range': {'detect_process': {'lt': 99}}},
{'term': {'task_type': 'detect'}}
]
}
}
}
try:
task_exist_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits']
except:
task_exist_result = []
task_exist_count = len(task_exist_result)
return task_exist_count
def show_detect_task():
results = []
query = [{'match':{'task_type': 'detect'}}]
try:
search_results = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
body={'query':{'bool':{'must':query}}, 'sort':[{'submit_date': 'desc'}], 'size':MAX_VALUE})['hits']['hits']
except:
search_results = []
for group_item in search_results:
source = group_item['_source']
task_name = source['task_name']
submit_date = ts2datetime(int(source['submit_date']))
submit_user = source['submit_user']
detect_type = source['detect_type']
state = source['state']
process = source['detect_process']
results.append([task_name, submit_user, submit_date, detect_type, state, process])
return results
def get_group_analysis(submit_user):
results = []
#step1: get query body
query_body = {
'query':{
'filtered':{
'filter':{
'bool':{
'must':[
{'term': {'submit_user': submit_user}},
{'term': {'task_type': 'analysis'}}
]
}
}
}
},
'sort': [{'submit_date': {'order': 'desc'}}],
'size': MAX_VALUE
}
#step2: search
try:
group_task_result = es_group_result.search(index=group_index_name, doc_type=group_index_type,\
body=query_body)['hits']['hits']
except:
group_task_result = []
#step3: task results
for group_item in group_task_result:
source = group_item['_source']
task_name = source['task_name']
if not task_name:
continue
task_status = source['status']
submit_ts = source['submit_date']
submit_date = ts2date(submit_ts)
try:
state = source['state']
except:
state = ''
results.append([task_name, submit_date, state, task_status])
return results
def search_task(task_name, submit_date, state, status, submit_user):
results = []
query = []
condition_num = 0
if task_name:
task_name_list = task_name.split(' ')
for item in task_name_list:
query.append({'wildcard': {'task_name': '*' + item + '*'}})
condition_num += 1
if submit_date:
submit_date_ts = datetime2ts(submit_date)
submit_date_start = submit_date_ts
submit_date_end = submit_date_ts + DAY
query.append({
'range': {
'submit_date': {
'gte': submit_date_start,
'lt': submit_date_end
}
}
})
condition_num += 1
if state:
state_list = state.split(' ')
for item in state_list:
query.append({'wildcard': {'state': '*' + item + '*'}})
condition_num += 1
if status:
query.append({'match': {'status': status}})
condition_num += 1
if submit_user:
query.append({'term': {'submit_user': submit_user}})
condition_num += 1
if condition_num > 0:
query.append({'term': {'task_type': 'analysis'}})
try:
source = es_group_result.search(index=group_index_name,
doc_type=group_index_type,
body={
'query': {
'bool': {
'must': query
}
},
'sort': [{
'count': {
'order': 'desc'
}
}],
'size':
MAX_VALUE
})
except Exception as e:
raise e
else:
query.append({'term': {'task_type': 'analysis'}})
source = es.search(index=group_index_name,
doc_type=group_index_type,
body={
'query': {
'bool': {
'must': query
}
},
'sort': [{
'count': {
'order': 'desc'
}
}],
'size': MAX_VALUE
})
try:
task_dict_list = source['hits']['hits']
except:
return None
result = []
for task_dict in task_dict_list:
try:
state = task_dict['_source']['state']
except:
state = ''
try:
status = task_dict['_source']['status']
except:
status = 0
result.append([
task_dict['_source']['task_name'],
task_dict['_source']['submit_date'], task_dict['_source']['count'],
state, status
])
return result
def search_task(task_name, submit_date, state, status, submit_user):
results = []
query = []
condition_num = 0
if task_name:
task_name_list = task_name.split(' ')
for item in task_name_list:
query.append({'wildcard':{'task_name': '*' + item + '*'}})
condition_num += 1
if submit_date:
submit_date_ts = datetime2ts(submit_date)
submit_date_start = submit_date_ts
submit_date_end = submit_date_ts + DAY
query.append({'range':{'submit_date': {'gte': submit_date_start, 'lt': submit_date_end}}})
condition_num += 1
if state:
state_list = state.split(' ')
for item in state_list:
query.append({'wildcard':{'state': '*' + item + '*'}})
condition_num += 1
if status:
query.append({'match':{'status': status}})
condition_num += 1
if submit_user:
query.append({'term':{'submit_user': submit_user}})
condition_num += 1
if condition_num > 0:
query.append({'term':{'task_type': 'analysis'}})
try:
source = es_group_result.search(
index = group_index_name,
doc_type = group_index_type,
body = {
'query':{
'bool':{
'must':query
}
},
'sort': [{'count':{'order': 'desc'}}],
'size': MAX_VALUE
}
)
except Exception as e:
raise e
else:
query.append({'term':{'task_type': 'analysis'}})
source = es.search(
index = group_index_name,
doc_type = group_index_type,
body = {
'query':{'bool':{
'must':query
}
},
'sort': [{'count': {'order': 'desc'}}],
'size': MAX_VALUE
}
)
try:
task_dict_list = source['hits']['hits']
except:
return None
result = []
for task_dict in task_dict_list:
try:
state = task_dict['_source']['state']
except:
state = ''
try:
status = task_dict['_source']['status']
except:
status = 0
result.append([task_dict['_source']['task_name'], task_dict['_source']['submit_date'], task_dict['_source']['count'], state, status])
return result