Exemplo n.º 1
0
def identify_exist_detect_task_count(submit_user):
    query_body = {
        'query': {
            'bool': {
                'must': [{
                    'term': {
                        'submit_user': submit_user
                    }
                }, {
                    'range': {
                        'detect_process': {
                            'lt': 99
                        }
                    }
                }, {
                    'term': {
                        'task_type': 'detect'
                    }
                }]
            }
        }
    }
    try:
        task_exist_result = es_group_result.search(
            index=group_index_name, doc_type=group_index_type,
            body=query_body)['hits']['hits']
    except:
        task_exist_result = []
    task_exist_count = len(task_exist_result)
    return task_exist_count
Exemplo n.º 2
0
def identify_exist_analysis_task_count(submit_user):
    query_body = {
        'query': {
            'bool': {
                'must': [{
                    'term': {
                        'submit_user': submit_user
                    }
                }, {
                    'term': {
                        'task_type': 'analysis'
                    }
                }, {
                    'term': {
                        'status': 0
                    }
                }]
            }
        }
    }
    try:
        task_exist_result = es_group_result.search(
            index=group_index_name, doc_type=group_index_type,
            body=query_body)['hits']['hits']
    except:
        task_exist_result = []
    task_exist_count = len(task_exist_result)
    return task_exist_count
Exemplo n.º 3
0
def get_group_detect(submit_user):
    results = []
    query_body = {
        'query':{
            'filtered':{
                'filter':{
                    'bool':{
                        'must':[
                            {'term': {'submit_user': submit_user}},
                            {'term': {'task_type': 'detect'}}
                            ]
                        }
                    }
                }
            },
        'sort':[{'submit_date': {'order': 'desc'}}],
        'size': MAX_VALUE
        }
    #search group task
    try:
        group_task_result = es_group_result.search(index=group_index_name, doc_type=group_index_type,\
                body=query_body)['hits']['hits']
    except:
        group_task_result = []
    #group task results
    for group_item in group_task_result:
        source = group_item['_source']
        task_name = source['task_name']
        task_process = source['detect_process']
        submit_ts = source['submit_date']
        submit_date = ts2date(submit_ts)
        state = source['state']
        task_type = source['detect_type']
        results.append([task_name, submit_date, state, task_type, task_process])
    return results
Exemplo n.º 4
0
def submit_task(input_data):
    print 'aaaaaaaaaaaaaaa'
    print input_data
    status = 0 # mark it can not submit
    task_name = input_data['task_name']
    submit_user = input_data['submit_user']
    task_id = submit_user + '-' + task_name
    #identify the compute task is not more than limit
    try:
        task_max_count = input_data['task_max_count']
    except:
        task_max_count = 0
    query_body = {
    'query':{
        'filtered':{
            'filter':{
                'bool':{
                    'must':[
                        {'term': {'submit_user': submit_user}},
                        {'term': {'status': 0}}
                        ]
                    }
                }
            }
        }
    }
    print es_group_result,group_index_name,group_index_type
    exist_compute_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits']
    exist_compute_count = len(exist_compute_result)
    if exist_compute_count >= task_max_count:
        return 'more than limit'
    #identify the task name is valid
    try:
        result = es_group_result.get(index=group_index_name, doc_type=group_index_type, id=task_id)['_source']
    except:
        status = 1
    if status != 0 and 'uid_file' not in input_data:
        input_data['status'] = 0 # mark the task not compute
        count = len(input_data['uid_list'])
        input_data['count'] = count
        input_data['task_type'] = 'analysis'
        input_data['submit_user'] = submit_user
        input_data['detect_type'] = ''
        input_data['detect_process'] = ''
        input_data['task_id'] = task_id
        add_es_dict = {'task_information': input_data, 'query_condition':''}
        # print es_group_result,group_index_name,group_index_type
        # print r
        # print '**********************8'
        es_group_result.index(index=group_index_name, doc_type=group_index_type, id=task_id, body=input_data)
        #print es_group_result,group_index_name,group_index_type
        r.lpush(group_analysis_queue_name, json.dumps(input_data))
        #print status
    print status
    return status
Exemplo n.º 5
0
def search_detect_task(task_name, submit_date, state, process, detect_type, submit_user):
    results = []
    query = [{'match':{'task_type': 'detect'}}]
    condition_num = 0
    if task_name:
        task_name_list = task_name.split(' ')
        for item in task_name_list:
            query.append({'wildcard':{'task_name': '*'+item+'*'}})
            condition_num += 1
    if submit_date:
        submit_date_ts = datetime2ts(submit_date)
        submit_date_from = submit_date_ts
        submit_date_to = submit_date_ts + DAY
        query.append({'range':{'submit_date':{'gte':submit_date_from, 'lt':submit_date_to}}})
        condition_num += 1
    if state:
        state_list = state.split(' ')
        for item in state_list:
            query.append({'wildcard':{'state': '*'+item+'*'}})
            condition_num += 1
    if process:
        query.append({'range':{'detect_process':{'from': int(process), 'to': MAX_PROCESS}}})
        condition_num += 1
    if detect_type:
        
        detect_type_list = detect_type.split(',')
        nest_body_list = []
        for type_item in detect_type_list:
            nest_body_list.append({'wildcard':{'detect_type': '*'+type_item+'*'}})
        query.append({'bool':{'should': nest_body_list}})
        
        condition_num += 1
    if submit_user:
        query.append({'term':{'submit_user': submit_user}})
        condition_num += 1
    try:
        search_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
                    body={'query':{'bool': {'must': query}}, 'sort':[{'submit_date': {'order': 'desc'}}], 'size':MAX_VALUE})['hits']['hits']
    except:
        search_result = []
    #get group information table
    for group_item in search_result:
        source = group_item['_source']
        task_name = source['task_name']
        submit_date = ts2datetime(int(source['submit_date']))
        submit_user = source['submit_user']
        detect_type = source['detect_type']
        state = source['state']
        process = source['detect_process']

        results.append([task_name, submit_user, submit_date, detect_type, state, process])
        
    return results
Exemplo n.º 6
0
def search_detect_task(task_name, submit_date, state, process, detect_type, submit_user):
    results = []
    query = [{'match':{'task_type': 'detect'}}]
    condition_num = 0
    if task_name:
        task_name_list = task_name.split(' ')
        for item in task_name_list:
            query.append({'wildcard':{'task_name': '*'+item+'*'}})
            condition_num += 1
    if submit_date:
        submit_date_ts = datetime2ts(submit_date)
        submit_date_from = submit_date_ts
        submit_date_to = submit_date_ts + DAY
        query.append({'range':{'submit_date':{'gte':submit_date_from, 'lt':submit_date_to}}})
        condition_num += 1
    if state:
        state_list = state.split(' ')
        for item in state_list:
            query.append({'wildcard':{'state': '*'+item+'*'}})
            condition_num += 1
    if process:
        query.append({'range':{'detect_process':{'from': int(process), 'to': MAX_PROCESS}}})
        condition_num += 1
    if detect_type:
        
        detect_type_list = detect_type.split(',')
        nest_body_list = []
        for type_item in detect_type_list:
            nest_body_list.append({'wildcard':{'detect_type': '*'+type_item+'*'}})
        query.append({'bool':{'should': nest_body_list}})
        
        condition_num += 1
    if submit_user:
        query.append({'wildcard':{'submit_user': '******'+submit_user+'*'}})
        condition_num += 1
    try:
        search_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
                    body={'query':{'bool': {'must': query}}, 'sort':[{'submit_date': {'order': 'desc'}}], 'size':MAX_VALUE})['hits']['hits']
    except:
        search_result = []
    #get group information table
    for group_item in search_result:
        source = group_item['_source']
        task_name = source['task_name']
        submit_date = ts2datetime(int(source['submit_date']))
        submit_user = source['submit_user']
        detect_type = source['detect_type']
        state = source['state']
        process = source['detect_process']

        results.append([task_name, submit_user, submit_date, detect_type, state, process])
        
    return results
Exemplo n.º 7
0
def get_group_analysis(submit_user):
    results = []
    #step1: get query body
    query_body = {
        'query': {
            'filtered': {
                'filter': {
                    'bool': {
                        'must': [{
                            'term': {
                                'submit_user': submit_user
                            }
                        }, {
                            'term': {
                                'task_type': 'analysis'
                            }
                        }]
                    }
                }
            }
        },
        'sort': [{
            'submit_date': {
                'order': 'desc'
            }
        }],
        'size': MAX_VALUE
    }
    #step2: search
    try:
        group_task_result = es_group_result.search(index=group_index_name, doc_type=group_index_type,\
                body=query_body)['hits']['hits']
    except:
        group_task_result = []
    #step3: task results
    for group_item in group_task_result:
        source = group_item['_source']
        task_name = source['task_name']
        if not task_name:
            continue
        task_status = source['status']
        submit_ts = source['submit_date']
        submit_date = ts2date(submit_ts)
        try:
            state = source['state']
        except:
            state = ''
        results.append([task_name, submit_date, state, task_status])

    return results
Exemplo n.º 8
0
def submit_task(input_data):
    status = 0 # mark it can not submit
    task_name = input_data['task_name']
    submit_user = input_data['submit_user']
    task_id = submit_user + '-' + task_name
    #identify the compute task is not more than limit
    try:
        task_max_count = input_data['task_max_count']
    except:
        task_max_count = 0
    query_body = {
    'query':{
        'filtered':{
            'filter':{
                'bool':{
                    'must':[
                        {'term': {'submit_user': submit_user}},
                        {'term': {'status': 0}}
                        ]
                    }
                }
            }
        }
    }
    exist_compute_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits']
    exist_compute_count = len(exist_compute_result)
    if exist_compute_count >= task_max_count:
        return 'more than limit'
    #identify the task name is valid
    try:
        result = es_group_result.get(index=group_index_name, doc_type=group_index_type, id=task_id)['_source']
    except:
        status = 1
    
    if status != 0 and 'uid_file' not in input_data:
        input_data['status'] = 0 # mark the task not compute
        count = len(input_data['uid_list'])
        input_data['count'] = count
        input_data['task_type'] = 'analysis'
        input_data['submit_user'] = submit_user
        input_data['detect_type'] = ''
        input_data['detect_process'] = ''
        add_es_dict = {'task_information': input_data, 'query_condition':''}
        es_group_result.index(index=group_index_name, doc_type=group_index_type, id=task_id, body=input_data)
        r.lpush(group_analysis_queue_name, json.dumps(input_data))
    
    return status
Exemplo n.º 9
0
def get_group_detect(submit_user):
    results = []
    query_body = {
        'query': {
            'filtered': {
                'filter': {
                    'bool': {
                        'must': [{
                            'term': {
                                'submit_user': submit_user
                            }
                        }, {
                            'term': {
                                'task_type': 'detect'
                            }
                        }]
                    }
                }
            }
        },
        'sort': [{
            'submit_date': {
                'order': 'desc'
            }
        }],
        'size': MAX_VALUE
    }
    #search group task
    try:
        group_task_result = es_group_result.search(index=group_index_name, doc_type=group_index_type,\
                body=query_body)['hits']['hits']
    except:
        group_task_result = []
    #group task results
    for group_item in group_task_result:
        source = group_item['_source']
        task_name = source['task_name']
        task_process = source['detect_process']
        submit_ts = source['submit_date']
        submit_date = ts2date(submit_ts)
        state = source['state']
        task_type = source['detect_type']
        results.append(
            [task_name, submit_date, state, task_type, task_process])
    return results
Exemplo n.º 10
0
def identify_exist_analysis_task_count(submit_user):
    query_body = {
    'query':{
        'bool':{
            'must':[
                {'term':{'submit_user': submit_user}},
                {'term': {'task_type': 'analysis'}},
                {'term': {'status': 0}}
            ]
            }
        }
    }
    try:
        task_exist_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits']
    except:
        task_exist_result = []
    task_exist_count = len(task_exist_result)
    return task_exist_count
Exemplo n.º 11
0
def identify_exist_detect_task_count(submit_user):
    query_body = {
    'query':{
            'bool':{
               'must':[
                    {'term': {'submit_user':submit_user}},
                    {'range': {'detect_process': {'lt': 99}}},
                    {'term': {'task_type': 'detect'}}
                    ]
                }
        }
    }
    try:
        task_exist_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, body=query_body)['hits']['hits']
    except:
        task_exist_result = []
    task_exist_count = len(task_exist_result)
    return task_exist_count
Exemplo n.º 12
0
def show_detect_task():
    results = []
    query = [{'match':{'task_type': 'detect'}}]
    try:
        search_results = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
                body={'query':{'bool':{'must':query}}, 'sort':[{'submit_date': 'desc'}], 'size':MAX_VALUE})['hits']['hits']
    except:
        search_results = []
    for group_item in search_results:
        source = group_item['_source']
        task_name = source['task_name']
        submit_date = ts2datetime(int(source['submit_date']))
        submit_user = source['submit_user']
        detect_type = source['detect_type']
        state = source['state']
        process = source['detect_process']
        results.append([task_name, submit_user, submit_date, detect_type, state, process])

    return results
Exemplo n.º 13
0
def get_group_analysis(submit_user):
    results = []
    #step1: get query body
    query_body = {
        'query':{
            'filtered':{
                'filter':{
                    'bool':{
                        'must':[
                            {'term': {'submit_user': submit_user}},
                            {'term': {'task_type': 'analysis'}}
                            ]
                        }
                    }
                }
            },
        'sort': [{'submit_date': {'order': 'desc'}}],
        'size': MAX_VALUE
        }
    #step2: search
    try:
        group_task_result = es_group_result.search(index=group_index_name, doc_type=group_index_type,\
                body=query_body)['hits']['hits']
    except:
        group_task_result = []
    #step3: task results
    for group_item in group_task_result:
        source = group_item['_source']
        task_name = source['task_name']
        if not task_name:
            continue
        task_status = source['status']
        submit_ts = source['submit_date']
        submit_date = ts2date(submit_ts)
        try:
            state = source['state']
        except:
            state = ''
        results.append([task_name, submit_date, state, task_status])

    return results
Exemplo n.º 14
0
def search_task(task_name, submit_date, state, status, submit_user):
    results = []
    query = []
    condition_num = 0
    if task_name:
        task_name_list = task_name.split(' ')
        for item in task_name_list:
            query.append({'wildcard': {'task_name': '*' + item + '*'}})
            condition_num += 1
    if submit_date:
        submit_date_ts = datetime2ts(submit_date)
        submit_date_start = submit_date_ts
        submit_date_end = submit_date_ts + DAY
        query.append({
            'range': {
                'submit_date': {
                    'gte': submit_date_start,
                    'lt': submit_date_end
                }
            }
        })
        condition_num += 1
    if state:
        state_list = state.split(' ')
        for item in state_list:
            query.append({'wildcard': {'state': '*' + item + '*'}})
            condition_num += 1
    if status:
        query.append({'match': {'status': status}})
        condition_num += 1
    if submit_user:
        query.append({'term': {'submit_user': submit_user}})
        condition_num += 1
    if condition_num > 0:
        query.append({'term': {'task_type': 'analysis'}})
        try:
            source = es_group_result.search(index=group_index_name,
                                            doc_type=group_index_type,
                                            body={
                                                'query': {
                                                    'bool': {
                                                        'must': query
                                                    }
                                                },
                                                'sort': [{
                                                    'count': {
                                                        'order': 'desc'
                                                    }
                                                }],
                                                'size':
                                                MAX_VALUE
                                            })
        except Exception as e:
            raise e
    else:
        query.append({'term': {'task_type': 'analysis'}})
        source = es.search(index=group_index_name,
                           doc_type=group_index_type,
                           body={
                               'query': {
                                   'bool': {
                                       'must': query
                                   }
                               },
                               'sort': [{
                                   'count': {
                                       'order': 'desc'
                                   }
                               }],
                               'size': MAX_VALUE
                           })

    try:
        task_dict_list = source['hits']['hits']
    except:
        return None
    result = []
    for task_dict in task_dict_list:
        try:
            state = task_dict['_source']['state']
        except:
            state = ''
        try:
            status = task_dict['_source']['status']
        except:
            status = 0
        result.append([
            task_dict['_source']['task_name'],
            task_dict['_source']['submit_date'], task_dict['_source']['count'],
            state, status
        ])

    return result
Exemplo n.º 15
0
def search_task(task_name, submit_date, state, status, submit_user):
    results = []
    query = []
    condition_num = 0
    if task_name:
        task_name_list = task_name.split(' ')
        for item in task_name_list:
            query.append({'wildcard':{'task_name': '*' + item + '*'}})
            condition_num += 1
    if submit_date:
        submit_date_ts = datetime2ts(submit_date)
        submit_date_start = submit_date_ts
        submit_date_end = submit_date_ts + DAY
        query.append({'range':{'submit_date': {'gte': submit_date_start, 'lt': submit_date_end}}})
        condition_num += 1
    if state:
        state_list = state.split(' ')
        for item in state_list:
            query.append({'wildcard':{'state': '*' + item + '*'}})
            condition_num += 1
    if status:
        query.append({'match':{'status': status}})
        condition_num += 1
    if submit_user:
        query.append({'term':{'submit_user': submit_user}})
        condition_num += 1
    if condition_num > 0:
        query.append({'term':{'task_type': 'analysis'}})
        try:
            source = es_group_result.search(
                    index = group_index_name,
                    doc_type = group_index_type,
                    body = {
                        'query':{
                            'bool':{
                                'must':query
                                }
                            },
                        'sort': [{'count':{'order': 'desc'}}],
                        'size': MAX_VALUE
                        }
                    )
        except Exception as e:
            raise e
    else:
        query.append({'term':{'task_type': 'analysis'}})
        source = es.search(
                index = group_index_name,
                doc_type = group_index_type,
                body = {
                    'query':{'bool':{
                        'must':query
                        }
                        },
                    'sort': [{'count': {'order': 'desc'}}],
                    'size': MAX_VALUE
                    }
                )

    try:
        task_dict_list = source['hits']['hits']
    except:
        return None
    result = []
    for task_dict in task_dict_list:
        try:
            state = task_dict['_source']['state']
        except:
            state = ''
        try:
            status = task_dict['_source']['status']
        except:
            status = 0
        result.append([task_dict['_source']['task_name'], task_dict['_source']['submit_date'], task_dict['_source']['count'], state, status])
    
    return result