def delete_attribute(attribute_name):
status = False
try:
result = es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['_source']
except:
return status
attribute_value = json.loads(result['value'])
es.delete(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)
# delete attribute in user_portrait
query = []
for value in attribute_value:
query.append({'match':{attribute_name: value}})
try:
attribute_user_result = es.search(index=user_index_name, doc_type=user_index_type, \
body={'query':{'bool':{'must':query}}})['hits']['hits']
except:
attribute_user_result = []
if attribute_user_result==[]:
status = True
return status
bulk_action = []
for user_dict in attribute_user_result:
try:
user_item = user_dict['_source']
except:
next
user_item.pop(attribute)
user = user_item['uid']
action = {'index':{'_id':str(user)}}
bulk_action.extend([action, user_item])
es.bulk(bulk_action, index=user_index_name, doc_type=index_type)
status = True
return status
def delete_group_results(task_name, submit_user):
task_id = submit_user + '-' + task_name
#step1: get group uid list
try:
group_result = es_group_result.get(index=group_index_name, doc_type=group_index_type,\
id=task_id)['_source']
except:
return False
uid_list = group_result['uid_list']
#step2: update group_tag in user_portrait
query_body = {'query': {'term': {'group': task_id}}}
try:
user_portrait_result = es_user_portrait.mget(index=portrait_index_name, doc_type=portrait_index_type,\
body={'ids': uid_list})['docs']
except:
user_portrait_result = []
bulk_action = []
for item in user_portrait_result:
uid = item['_id']
if item['found'] == True:
try:
source = item['_source']
except:
source = {}
try:
group_tag = source['group']
except:
group_tag = ''
if group_tag != '':
new_group_tag_list = []
group_tag_list = group_tag.split('&')
for group_tag_item in group_tag_list:
if group_tag_item != task_id and group_tag_item != '[email protected]':
new_group_tag_list.append(group_tag_item)
new_group_tag = '&'.join(new_group_tag_list)
else:
new_group_tag = ''
action = {'update': {'_id': uid}}
bulk_action.extend([action, {'doc': {'group': new_group_tag}}])
if bulk_action:
print 'bulk_action:', bulk_action
es_user_portrait.bulk(bulk_action,
index=portrait_index_name,
doc_type=portrait_index_type)
#step3: delete group results in group_manage
try:
print 'yes delete'
result = es.delete(index=index_name, doc_type=index_type, id=task_id)
except:
return False
return True
def delete_group_results(task_name, submit_user):
task_id = submit_user + '-' + task_name
#step1: get group uid list
try:
group_result = es_group_result.get(index=group_index_name, doc_type=group_index_type,\
id=task_id)['_source']
except:
return False
uid_list = group_result['uid_list']
#step2: update group_tag in user_portrait
query_body = {'query':{'term':{'group': task_id}}}
try:
user_portrait_result = es_user_portrait.mget(index=portrait_index_name, doc_type=portrait_index_type,\
body={'ids': uid_list})['docs']
except:
user_portrait_result = []
bulk_action = []
for item in user_portrait_result:
uid = item['_id']
if item['found'] == True:
try:
source = item['_source']
except:
source = {}
try:
group_tag = source['group']
except:
group_tag = ''
if group_tag != '':
new_group_tag_list = []
group_tag_list = group_tag.split('&')
for group_tag_item in group_tag_list:
if group_tag_item != task_id and group_tag_item != '[email protected]':
new_group_tag_list.append(group_tag_item)
new_group_tag = '&'.join(new_group_tag_list)
else:
new_group_tag = ''
action = {'update':{'_id': uid}}
bulk_action.extend([action, {'doc': {'group': new_group_tag}}])
if bulk_action:
print 'bulk_action:', bulk_action
es_user_portrait.bulk(bulk_action, index=portrait_index_name, doc_type=portrait_index_type)
#step3: delete group results in group_manage
try:
print 'yes delete'
result = es.delete(index=index_name, doc_type=index_type, id=task_id)
except:
return False
return True
def delete_es(task_name):
s_re = scan(es, query={"query":{"match_all":{}},"size":1000}, index=index_sensing_task, doc_type=task_name)
bulk_action = []
count = 0
search_list = []
uid_list = []
while 1:
try:
uid = s_re.next()['_id']
count += 1
action = {"delete": {"_index": index_sensing_task, "_type": task_name, "_id": uid}}
bulk_action.append(action)
if count % 100 == 0:
es.bulk(bulk_action, index=index_sensing_task, doc_type=task_name, timeout=60)
bulk_action = []
except StopIteration:
print "all done"
if bulk_action:
es.bulk(bulk_action, index=index_sensing_task, doc_type=task_name, timeout=60)
break
except Exception, r:
print Exception, r
bulk_action = []
count = 0
for user_dict in attribute_user_result:
try:
user_item = user_dict['_source']
except:
next
tmp = user_item[submit_user_tag]
delete_set = set(tmp.split('&')) - set(portrait_attribute_field)
user_item[submit_user_tag] = "&".join(list(delete_set))
user = user_item['uid']
action = {'index':{'_id':str(user)}}
bulk_action.extend([action, user_item])
count += 1
if count % 1000 == 0:
es.bulk(bulk_action, index=user_index_name, doc_type=user_index_type)
bulk_action = []
if bulk_action:
es.bulk(bulk_action, index=user_index_name, doc_type=user_index_type)
status = True
return status
# use to add attribute to user in es_user_portrait
def add_attribute_portrait(uid, attribute_name, attribute_value, submit_user):
status = False
submit_user_tag = submit_user + "-tag"
id_attribute = submit_user + "-" + attribute_name
add_attribute_value = attribute_name + "-" + attribute_value
# identify the user exist
# identify the attribute exist
except:
attribute_user_result = []
if attribute_user_result == []:
status = True
return status
bulk_action = []
for user_dict in attribute_user_result:
try:
user_item = user_dict['_source']
except:
next
user_item.pop(attribute_name)
user = user_item['uid']
action = {'index': {'_id': str(user)}}
bulk_action.extend([action, user_item])
es.bulk(bulk_action, index=user_index_name, doc_type=user_index_type)
status = True
return status
# use to add attribute to user in es_user_portrait
def add_attribute_portrait(uid, attribute_name, attribute_value, submit_user):
status = False
# identify the user exist
# identify the attribute exist
# identify the attribute exist in user_portrait
# add attribute in user_portrait
try:
user_result = es.get(index=user_index_name,
doc_type=user_index_type,
while 1:
try:
uid = s_re.next()['_id']
count += 1
action = {"delete": {"_index": index_sensing_task, "_type": task_name, "_id": uid}}
bulk_action.append(action)
if count % 100 == 0:
es.bulk(bulk_action, index=index_sensing_task, doc_type=task_name, timeout=60)
bulk_action = []
except StopIteration:
print "all done"
if bulk_action:
es.bulk(bulk_action, index=index_sensing_task, doc_type=task_name, timeout=60)
break
except Exception, r:
print Exception, r
if bulk_action:
es.bulk(bulk_action, index=index_sensing_task, doc_type=task_name, timeout=60)
print count
return "1"
if __name__ == "__main__":
delete_es("律师群体言论".decode('utf-8'))
delete_es("洪水灾情".decode('utf-8'))
delete_es("民主言论".decode('utf-8'))
delete_es("媒体感知社会事件".decode('utf-8'))
