def upgrade_downgrade(action):
"""Upgrade or downgrade index holdings.
Correct items_count and public_items_count for holdings of type serial.
:param str action: upgrade or downgrade.
"""
index = HoldingsSearch.Meta.index
query = HoldingsSearch()\
.filter('term', holdings_type='serial') \
.source(['pid'])
ids = [(h.meta.id, h.pid) for h in query.scan()]
count = 0
LOGGER.info(f'Indexing {len(ids)} records ....')
for (_id, pid) in ids:
document = Document.get(_id, index=index, using=current_search_client)
items_count, public_items_count = get_counts(pid, action)
document.update(items_count=items_count,
public_items_count=public_items_count,
index=index,
using=current_search_client,
refresh=True)
count += 1
LOGGER.info(f'{count} records indexed.')
def update(cls, _id, date, data):
"""Update all data for a record.
:param str _id: Elasticsearch document ID.
:param str date: Log date, useful for getting the right index.
:param dict data: New record data.
"""
index = cls.get_index({'date': date})
document = Document.get(_id, index=index, using=current_search_client)
# Assign each properties to the document
for key, item in data.items():
document[key] = item
result = document.save(
index=index,
using=current_search_client,
refresh=True,
)
if result != 'updated':
raise Exception('Operation log cannot be updated.')
def search_by_id(self, id):
return Document.get(id, self.__es, self.__index)
def get(self, request):
id = request.GET.get("id", "")
doc = Document.get(id=id, index='policynew', using=client).to_dict()
link = doc['link']
title = doc['title']
policy_object_list = []
# 从数据库中取出推荐的政策
policy_recommend = PolicyRecommend.objects.filter(title=title)
if len(policy_recommend) > 0:
policy_object = policy_recommend[0]
policy_str = policy_object.recommend
policy_items = policy_str.split(",")[:3]
policy_names = [item.split(' ')[0] for item in policy_items]
policy_scores = [
item.split(' ')[1] for item in policy_items
if len(item.split(' ')[1]) == 7
]
print(policy_scores)
policy_recommend_urls = []
for policy_name in policy_names:
resp = Search(using=client,
index='policynew').query("match",
title=policy_name)
response = resp.execute()
link_str = response['hits']['hits'][0]["_id"]
if (policy_name == response[0].title):
policy_recommend_url = "http://127.0.0.1:8000/detail/?id=" + link_str
else:
policy_recommend_url = ""
policy_recommend_urls.append(policy_recommend_url)
policy_object_list = list(
zip(policy_names, policy_scores, policy_recommend_urls))
# relation_policies_dict[relation_policy] = relation_policy_url
# 从数据库中取出相关的政策
relation_policies_dict = {}
relations = RelationPolicies.objects.filter(title=title)
if len(relations) == 1:
# print(relations[0].relation_policies)
relation_policies = set(relations[0].relation_policies.split(','))
# print(relation_policies)
for relation_policy in relation_policies:
if (relation_policy != ''):
resp = Search(using=client, index='policynew').query(
"match", title=relation_policy)
response = resp.execute()
link_str = response['hits']['hits'][0]["_id"]
if (relation_policy == response[0].title):
relation_policy_url = "http://127.0.0.1:8000/detail/?id=" + link_str
else:
relation_policy_url = ""
relation_policies_dict[
relation_policy] = relation_policy_url
# 生成结果返回对象
result_dict = {}
result_dict['link'] = link
if policy_object_list != []:
print(policy_object_list)
result_dict['policy_object_list'] = policy_object_list
if relation_policies_dict != {}:
result_dict['relation_policies_dict'] = relation_policies_dict
return render(
request, 'detail.html', {
'link': link,
'relation_policies_dict': relation_policies_dict,
'policy_object_list': policy_object_list
})
def merge_generated_parameters(params, idx, hash):
"""
:param params:
:param paramsfile:
:param idx:
:return:
"""
layer_id = "%s_%s" % (hash, socket.gethostname())
es = Elasticsearch(
current_app.config.get("ELASTIC").split(","),
verify_certs=False,
timeout=120
)
#See if the hash exists
try:
doc = Document.get(id=layer_id, using=es, index=".datashader_layers")
except NotFoundError:
doc = None
if not doc:
#if not, create the hash in the db but only if it does not already exist
try:
doc = Document(_id=layer_id,
creating_host=socket.gethostname(),
creating_pid=os.getpid(),
creating_timestamp=datetime.now(),
generated_params=None,
params=params)
doc.save(using=es, index=".datashader_layers", op_type="create", skip_empty=False)
current_app.logger.debug("Created Hash document")
except ConflictError:
current_app.logger.debug("Hash document now exists, continuing")
#re-fetch to get sequence number correct
doc = Document.get(id=layer_id, using=es, index=".datashader_layers")
#Check for generator timeouts:
if doc.to_dict().get("generated_params", {}).get("generation_start_time") and \
datetime.now() > datetime.strptime(doc.to_dict().get("generated_params", {}).get("generation_start_time"),"%Y-%m-%dT%H:%M:%S.%f")+timedelta(seconds=5*60):
#Something caused the worker generating the params to time out so clear that entry
try:
doc.update(using=es, index=".datashader_layers", retry_on_conflict=0, refresh=True, \
generated_params=None)
except ConflictError:
current_app.logger.debug("Abandoned resetting parameters due to conflict, other process has completed.")
#Loop-check if the generated params are in missing/in-process/complete
timeout_at = datetime.now()+timedelta(seconds=45)
while doc.to_dict().get("generated_params", {}).get("complete", False) == False:
if datetime.now() > timeout_at:
current_app.logger.info("Hit timeout waiting for generated parameters to be placed into database")
break
#If missing, mark them as in generation
if not doc.to_dict().get("generated_params", None):
#Mark them as being generated but do so with concurrenty control
#https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html
current_app.logger.info("Discovering generated parameters")
generated_params = dict()
generated_params["complete"] = False
generated_params["generation_start_time"] = datetime.now()
generated_params["generating_host"] = socket.gethostname()
generated_params["generating_pid"] = os.getpid()
try:
doc.update(using=es, index=".datashader_layers", retry_on_conflict=0, refresh=True, \
generated_params=generated_params)
except ConflictError:
current_app.logger.debug("Abandoned generating parameters due to conflict, will wait for other process to complete.")
break
#Generate and save off parameters
current_app.logger.warn("Discovering generated params")
generated_params.update(generate_global_params(params, idx))
generated_params["generation_complete_time"] = datetime.now()
generated_params["complete"] = True
#Store off generated params
doc.update(using=es, index=".datashader_layers", retry_on_conflict=0, refresh=True, \
generated_params=generated_params)
break
else:
time.sleep(1)
doc = Document.get(id=layer_id, using=es, index=".datashader_layers")
#We now have params so use them
params["generated_params"] = doc.to_dict().get("generated_params")
return params
