def identify_exist_detect_task_count(submit_user): query_body = { 'query': { 'bool': { 'must': [{ 'term': { 'submit_user': submit_user } }, { 'range': { 'detect_process': { 'lt': 99 } } }, { 'term': { 'task_type': 'detect' } }] } } } try: task_exist_result = es_group_result.search( index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits'] except: task_exist_result = [] task_exist_count = len(task_exist_result) return task_exist_count
def identify_exist_analysis_task_count(submit_user): query_body = { 'query': { 'bool': { 'must': [{ 'term': { 'submit_user': submit_user } }, { 'term': { 'task_type': 'analysis' } }, { 'term': { 'status': 0 } }] } } } try: task_exist_result = es_group_result.search( index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits'] except: task_exist_result = [] task_exist_count = len(task_exist_result) return task_exist_count
def get_group_detect(submit_user): results = [] query_body = { 'query':{ 'filtered':{ 'filter':{ 'bool':{ 'must':[ {'term': {'submit_user': submit_user}}, {'term': {'task_type': 'detect'}} ] } } } }, 'sort':[{'submit_date': {'order': 'desc'}}], 'size': MAX_VALUE } #search group task try: group_task_result = es_group_result.search(index=group_index_name, doc_type=group_index_type,\ body=query_body)['hits']['hits'] except: group_task_result = [] #group task results for group_item in group_task_result: source = group_item['_source'] task_name = source['task_name'] task_process = source['detect_process'] submit_ts = source['submit_date'] submit_date = ts2date(submit_ts) state = source['state'] task_type = source['detect_type'] results.append([task_name, submit_date, state, task_type, task_process]) return results
def submit_task(input_data): print 'aaaaaaaaaaaaaaa' print input_data status = 0 # mark it can not submit task_name = input_data['task_name'] submit_user = input_data['submit_user'] task_id = submit_user + '-' + task_name #identify the compute task is not more than limit try: task_max_count = input_data['task_max_count'] except: task_max_count = 0 query_body = { 'query':{ 'filtered':{ 'filter':{ 'bool':{ 'must':[ {'term': {'submit_user': submit_user}}, {'term': {'status': 0}} ] } } } } } print es_group_result,group_index_name,group_index_type exist_compute_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits'] exist_compute_count = len(exist_compute_result) if exist_compute_count >= task_max_count: return 'more than limit' #identify the task name is valid try: result = es_group_result.get(index=group_index_name, doc_type=group_index_type, id=task_id)['_source'] except: status = 1 if status != 0 and 'uid_file' not in input_data: input_data['status'] = 0 # mark the task not compute count = len(input_data['uid_list']) input_data['count'] = count input_data['task_type'] = 'analysis' input_data['submit_user'] = submit_user input_data['detect_type'] = '' input_data['detect_process'] = '' input_data['task_id'] = task_id add_es_dict = {'task_information': input_data, 'query_condition':''} # print es_group_result,group_index_name,group_index_type # print r # print '**********************8' es_group_result.index(index=group_index_name, doc_type=group_index_type, id=task_id, body=input_data) #print es_group_result,group_index_name,group_index_type r.lpush(group_analysis_queue_name, json.dumps(input_data)) #print status print status return status
def search_detect_task(task_name, submit_date, state, process, detect_type, submit_user): results = [] query = [{'match':{'task_type': 'detect'}}] condition_num = 0 if task_name: task_name_list = task_name.split(' ') for item in task_name_list: query.append({'wildcard':{'task_name': '*'+item+'*'}}) condition_num += 1 if submit_date: submit_date_ts = datetime2ts(submit_date) submit_date_from = submit_date_ts submit_date_to = submit_date_ts + DAY query.append({'range':{'submit_date':{'gte':submit_date_from, 'lt':submit_date_to}}}) condition_num += 1 if state: state_list = state.split(' ') for item in state_list: query.append({'wildcard':{'state': '*'+item+'*'}}) condition_num += 1 if process: query.append({'range':{'detect_process':{'from': int(process), 'to': MAX_PROCESS}}}) condition_num += 1 if detect_type: detect_type_list = detect_type.split(',') nest_body_list = [] for type_item in detect_type_list: nest_body_list.append({'wildcard':{'detect_type': '*'+type_item+'*'}}) query.append({'bool':{'should': nest_body_list}}) condition_num += 1 if submit_user: query.append({'term':{'submit_user': submit_user}}) condition_num += 1 try: search_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, \ body={'query':{'bool': {'must': query}}, 'sort':[{'submit_date': {'order': 'desc'}}], 'size':MAX_VALUE})['hits']['hits'] except: search_result = [] #get group information table for group_item in search_result: source = group_item['_source'] task_name = source['task_name'] submit_date = ts2datetime(int(source['submit_date'])) submit_user = source['submit_user'] detect_type = source['detect_type'] state = source['state'] process = source['detect_process'] results.append([task_name, submit_user, submit_date, detect_type, state, process]) return results
def search_detect_task(task_name, submit_date, state, process, detect_type, submit_user): results = [] query = [{'match':{'task_type': 'detect'}}] condition_num = 0 if task_name: task_name_list = task_name.split(' ') for item in task_name_list: query.append({'wildcard':{'task_name': '*'+item+'*'}}) condition_num += 1 if submit_date: submit_date_ts = datetime2ts(submit_date) submit_date_from = submit_date_ts submit_date_to = submit_date_ts + DAY query.append({'range':{'submit_date':{'gte':submit_date_from, 'lt':submit_date_to}}}) condition_num += 1 if state: state_list = state.split(' ') for item in state_list: query.append({'wildcard':{'state': '*'+item+'*'}}) condition_num += 1 if process: query.append({'range':{'detect_process':{'from': int(process), 'to': MAX_PROCESS}}}) condition_num += 1 if detect_type: detect_type_list = detect_type.split(',') nest_body_list = [] for type_item in detect_type_list: nest_body_list.append({'wildcard':{'detect_type': '*'+type_item+'*'}}) query.append({'bool':{'should': nest_body_list}}) condition_num += 1 if submit_user: query.append({'wildcard':{'submit_user': '******'+submit_user+'*'}}) condition_num += 1 try: search_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, \ body={'query':{'bool': {'must': query}}, 'sort':[{'submit_date': {'order': 'desc'}}], 'size':MAX_VALUE})['hits']['hits'] except: search_result = [] #get group information table for group_item in search_result: source = group_item['_source'] task_name = source['task_name'] submit_date = ts2datetime(int(source['submit_date'])) submit_user = source['submit_user'] detect_type = source['detect_type'] state = source['state'] process = source['detect_process'] results.append([task_name, submit_user, submit_date, detect_type, state, process]) return results
def get_group_analysis(submit_user): results = [] #step1: get query body query_body = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'term': { 'submit_user': submit_user } }, { 'term': { 'task_type': 'analysis' } }] } } } }, 'sort': [{ 'submit_date': { 'order': 'desc' } }], 'size': MAX_VALUE } #step2: search try: group_task_result = es_group_result.search(index=group_index_name, doc_type=group_index_type,\ body=query_body)['hits']['hits'] except: group_task_result = [] #step3: task results for group_item in group_task_result: source = group_item['_source'] task_name = source['task_name'] if not task_name: continue task_status = source['status'] submit_ts = source['submit_date'] submit_date = ts2date(submit_ts) try: state = source['state'] except: state = '' results.append([task_name, submit_date, state, task_status]) return results
def submit_task(input_data): status = 0 # mark it can not submit task_name = input_data['task_name'] submit_user = input_data['submit_user'] task_id = submit_user + '-' + task_name #identify the compute task is not more than limit try: task_max_count = input_data['task_max_count'] except: task_max_count = 0 query_body = { 'query':{ 'filtered':{ 'filter':{ 'bool':{ 'must':[ {'term': {'submit_user': submit_user}}, {'term': {'status': 0}} ] } } } } } exist_compute_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits'] exist_compute_count = len(exist_compute_result) if exist_compute_count >= task_max_count: return 'more than limit' #identify the task name is valid try: result = es_group_result.get(index=group_index_name, doc_type=group_index_type, id=task_id)['_source'] except: status = 1 if status != 0 and 'uid_file' not in input_data: input_data['status'] = 0 # mark the task not compute count = len(input_data['uid_list']) input_data['count'] = count input_data['task_type'] = 'analysis' input_data['submit_user'] = submit_user input_data['detect_type'] = '' input_data['detect_process'] = '' add_es_dict = {'task_information': input_data, 'query_condition':''} es_group_result.index(index=group_index_name, doc_type=group_index_type, id=task_id, body=input_data) r.lpush(group_analysis_queue_name, json.dumps(input_data)) return status
def get_group_detect(submit_user): results = [] query_body = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'term': { 'submit_user': submit_user } }, { 'term': { 'task_type': 'detect' } }] } } } }, 'sort': [{ 'submit_date': { 'order': 'desc' } }], 'size': MAX_VALUE } #search group task try: group_task_result = es_group_result.search(index=group_index_name, doc_type=group_index_type,\ body=query_body)['hits']['hits'] except: group_task_result = [] #group task results for group_item in group_task_result: source = group_item['_source'] task_name = source['task_name'] task_process = source['detect_process'] submit_ts = source['submit_date'] submit_date = ts2date(submit_ts) state = source['state'] task_type = source['detect_type'] results.append( [task_name, submit_date, state, task_type, task_process]) return results
def identify_exist_analysis_task_count(submit_user): query_body = { 'query':{ 'bool':{ 'must':[ {'term':{'submit_user': submit_user}}, {'term': {'task_type': 'analysis'}}, {'term': {'status': 0}} ] } } } try: task_exist_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits'] except: task_exist_result = [] task_exist_count = len(task_exist_result) return task_exist_count
def identify_exist_detect_task_count(submit_user): query_body = { 'query':{ 'bool':{ 'must':[ {'term': {'submit_user':submit_user}}, {'range': {'detect_process': {'lt': 99}}}, {'term': {'task_type': 'detect'}} ] } } } try: task_exist_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits'] except: task_exist_result = [] task_exist_count = len(task_exist_result) return task_exist_count
def show_detect_task(): results = [] query = [{'match':{'task_type': 'detect'}}] try: search_results = es_group_result.search(index=group_index_name, doc_type=group_index_type, \ body={'query':{'bool':{'must':query}}, 'sort':[{'submit_date': 'desc'}], 'size':MAX_VALUE})['hits']['hits'] except: search_results = [] for group_item in search_results: source = group_item['_source'] task_name = source['task_name'] submit_date = ts2datetime(int(source['submit_date'])) submit_user = source['submit_user'] detect_type = source['detect_type'] state = source['state'] process = source['detect_process'] results.append([task_name, submit_user, submit_date, detect_type, state, process]) return results
def get_group_analysis(submit_user): results = [] #step1: get query body query_body = { 'query':{ 'filtered':{ 'filter':{ 'bool':{ 'must':[ {'term': {'submit_user': submit_user}}, {'term': {'task_type': 'analysis'}} ] } } } }, 'sort': [{'submit_date': {'order': 'desc'}}], 'size': MAX_VALUE } #step2: search try: group_task_result = es_group_result.search(index=group_index_name, doc_type=group_index_type,\ body=query_body)['hits']['hits'] except: group_task_result = [] #step3: task results for group_item in group_task_result: source = group_item['_source'] task_name = source['task_name'] if not task_name: continue task_status = source['status'] submit_ts = source['submit_date'] submit_date = ts2date(submit_ts) try: state = source['state'] except: state = '' results.append([task_name, submit_date, state, task_status]) return results
def search_task(task_name, submit_date, state, status, submit_user): results = [] query = [] condition_num = 0 if task_name: task_name_list = task_name.split(' ') for item in task_name_list: query.append({'wildcard': {'task_name': '*' + item + '*'}}) condition_num += 1 if submit_date: submit_date_ts = datetime2ts(submit_date) submit_date_start = submit_date_ts submit_date_end = submit_date_ts + DAY query.append({ 'range': { 'submit_date': { 'gte': submit_date_start, 'lt': submit_date_end } } }) condition_num += 1 if state: state_list = state.split(' ') for item in state_list: query.append({'wildcard': {'state': '*' + item + '*'}}) condition_num += 1 if status: query.append({'match': {'status': status}}) condition_num += 1 if submit_user: query.append({'term': {'submit_user': submit_user}}) condition_num += 1 if condition_num > 0: query.append({'term': {'task_type': 'analysis'}}) try: source = es_group_result.search(index=group_index_name, doc_type=group_index_type, body={ 'query': { 'bool': { 'must': query } }, 'sort': [{ 'count': { 'order': 'desc' } }], 'size': MAX_VALUE }) except Exception as e: raise e else: query.append({'term': {'task_type': 'analysis'}}) source = es.search(index=group_index_name, doc_type=group_index_type, body={ 'query': { 'bool': { 'must': query } }, 'sort': [{ 'count': { 'order': 'desc' } }], 'size': MAX_VALUE }) try: task_dict_list = source['hits']['hits'] except: return None result = [] for task_dict in task_dict_list: try: state = task_dict['_source']['state'] except: state = '' try: status = task_dict['_source']['status'] except: status = 0 result.append([ task_dict['_source']['task_name'], task_dict['_source']['submit_date'], task_dict['_source']['count'], state, status ]) return result
def search_task(task_name, submit_date, state, status, submit_user): results = [] query = [] condition_num = 0 if task_name: task_name_list = task_name.split(' ') for item in task_name_list: query.append({'wildcard':{'task_name': '*' + item + '*'}}) condition_num += 1 if submit_date: submit_date_ts = datetime2ts(submit_date) submit_date_start = submit_date_ts submit_date_end = submit_date_ts + DAY query.append({'range':{'submit_date': {'gte': submit_date_start, 'lt': submit_date_end}}}) condition_num += 1 if state: state_list = state.split(' ') for item in state_list: query.append({'wildcard':{'state': '*' + item + '*'}}) condition_num += 1 if status: query.append({'match':{'status': status}}) condition_num += 1 if submit_user: query.append({'term':{'submit_user': submit_user}}) condition_num += 1 if condition_num > 0: query.append({'term':{'task_type': 'analysis'}}) try: source = es_group_result.search( index = group_index_name, doc_type = group_index_type, body = { 'query':{ 'bool':{ 'must':query } }, 'sort': [{'count':{'order': 'desc'}}], 'size': MAX_VALUE } ) except Exception as e: raise e else: query.append({'term':{'task_type': 'analysis'}}) source = es.search( index = group_index_name, doc_type = group_index_type, body = { 'query':{'bool':{ 'must':query } }, 'sort': [{'count': {'order': 'desc'}}], 'size': MAX_VALUE } ) try: task_dict_list = source['hits']['hits'] except: return None result = [] for task_dict in task_dict_list: try: state = task_dict['_source']['state'] except: state = '' try: status = task_dict['_source']['status'] except: status = 0 result.append([task_dict['_source']['task_name'], task_dict['_source']['submit_date'], task_dict['_source']['count'], state, status]) return result