def upgrade_downgrade(action): """Upgrade or downgrade index holdings. Correct items_count and public_items_count for holdings of type serial. :param str action: upgrade or downgrade. """ index = HoldingsSearch.Meta.index query = HoldingsSearch()\ .filter('term', holdings_type='serial') \ .source(['pid']) ids = [(h.meta.id, h.pid) for h in query.scan()] count = 0 LOGGER.info(f'Indexing {len(ids)} records ....') for (_id, pid) in ids: document = Document.get(_id, index=index, using=current_search_client) items_count, public_items_count = get_counts(pid, action) document.update(items_count=items_count, public_items_count=public_items_count, index=index, using=current_search_client, refresh=True) count += 1 LOGGER.info(f'{count} records indexed.')
def update(cls, _id, date, data): """Update all data for a record. :param str _id: Elasticsearch document ID. :param str date: Log date, useful for getting the right index. :param dict data: New record data. """ index = cls.get_index({'date': date}) document = Document.get(_id, index=index, using=current_search_client) # Assign each properties to the document for key, item in data.items(): document[key] = item result = document.save( index=index, using=current_search_client, refresh=True, ) if result != 'updated': raise Exception('Operation log cannot be updated.')
def search_by_id(self, id): return Document.get(id, self.__es, self.__index)
def get(self, request): id = request.GET.get("id", "") doc = Document.get(id=id, index='policynew', using=client).to_dict() link = doc['link'] title = doc['title'] policy_object_list = [] # 从数据库中取出推荐的政策 policy_recommend = PolicyRecommend.objects.filter(title=title) if len(policy_recommend) > 0: policy_object = policy_recommend[0] policy_str = policy_object.recommend policy_items = policy_str.split(",")[:3] policy_names = [item.split(' ')[0] for item in policy_items] policy_scores = [ item.split(' ')[1] for item in policy_items if len(item.split(' ')[1]) == 7 ] print(policy_scores) policy_recommend_urls = [] for policy_name in policy_names: resp = Search(using=client, index='policynew').query("match", title=policy_name) response = resp.execute() link_str = response['hits']['hits'][0]["_id"] if (policy_name == response[0].title): policy_recommend_url = "http://127.0.0.1:8000/detail/?id=" + link_str else: policy_recommend_url = "" policy_recommend_urls.append(policy_recommend_url) policy_object_list = list( zip(policy_names, policy_scores, policy_recommend_urls)) # relation_policies_dict[relation_policy] = relation_policy_url # 从数据库中取出相关的政策 relation_policies_dict = {} relations = RelationPolicies.objects.filter(title=title) if len(relations) == 1: # print(relations[0].relation_policies) relation_policies = set(relations[0].relation_policies.split(',')) # print(relation_policies) for relation_policy in relation_policies: if (relation_policy != ''): resp = Search(using=client, index='policynew').query( "match", title=relation_policy) response = resp.execute() link_str = response['hits']['hits'][0]["_id"] if (relation_policy == response[0].title): relation_policy_url = "http://127.0.0.1:8000/detail/?id=" + link_str else: relation_policy_url = "" relation_policies_dict[ relation_policy] = relation_policy_url # 生成结果返回对象 result_dict = {} result_dict['link'] = link if policy_object_list != []: print(policy_object_list) result_dict['policy_object_list'] = policy_object_list if relation_policies_dict != {}: result_dict['relation_policies_dict'] = relation_policies_dict return render( request, 'detail.html', { 'link': link, 'relation_policies_dict': relation_policies_dict, 'policy_object_list': policy_object_list })
def merge_generated_parameters(params, idx, hash): """ :param params: :param paramsfile: :param idx: :return: """ layer_id = "%s_%s" % (hash, socket.gethostname()) es = Elasticsearch( current_app.config.get("ELASTIC").split(","), verify_certs=False, timeout=120 ) #See if the hash exists try: doc = Document.get(id=layer_id, using=es, index=".datashader_layers") except NotFoundError: doc = None if not doc: #if not, create the hash in the db but only if it does not already exist try: doc = Document(_id=layer_id, creating_host=socket.gethostname(), creating_pid=os.getpid(), creating_timestamp=datetime.now(), generated_params=None, params=params) doc.save(using=es, index=".datashader_layers", op_type="create", skip_empty=False) current_app.logger.debug("Created Hash document") except ConflictError: current_app.logger.debug("Hash document now exists, continuing") #re-fetch to get sequence number correct doc = Document.get(id=layer_id, using=es, index=".datashader_layers") #Check for generator timeouts: if doc.to_dict().get("generated_params", {}).get("generation_start_time") and \ datetime.now() > datetime.strptime(doc.to_dict().get("generated_params", {}).get("generation_start_time"),"%Y-%m-%dT%H:%M:%S.%f")+timedelta(seconds=5*60): #Something caused the worker generating the params to time out so clear that entry try: doc.update(using=es, index=".datashader_layers", retry_on_conflict=0, refresh=True, \ generated_params=None) except ConflictError: current_app.logger.debug("Abandoned resetting parameters due to conflict, other process has completed.") #Loop-check if the generated params are in missing/in-process/complete timeout_at = datetime.now()+timedelta(seconds=45) while doc.to_dict().get("generated_params", {}).get("complete", False) == False: if datetime.now() > timeout_at: current_app.logger.info("Hit timeout waiting for generated parameters to be placed into database") break #If missing, mark them as in generation if not doc.to_dict().get("generated_params", None): #Mark them as being generated but do so with concurrenty control #https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html current_app.logger.info("Discovering generated parameters") generated_params = dict() generated_params["complete"] = False generated_params["generation_start_time"] = datetime.now() generated_params["generating_host"] = socket.gethostname() generated_params["generating_pid"] = os.getpid() try: doc.update(using=es, index=".datashader_layers", retry_on_conflict=0, refresh=True, \ generated_params=generated_params) except ConflictError: current_app.logger.debug("Abandoned generating parameters due to conflict, will wait for other process to complete.") break #Generate and save off parameters current_app.logger.warn("Discovering generated params") generated_params.update(generate_global_params(params, idx)) generated_params["generation_complete_time"] = datetime.now() generated_params["complete"] = True #Store off generated params doc.update(using=es, index=".datashader_layers", retry_on_conflict=0, refresh=True, \ generated_params=generated_params) break else: time.sleep(1) doc = Document.get(id=layer_id, using=es, index=".datashader_layers") #We now have params so use them params["generated_params"] = doc.to_dict().get("generated_params") return params