def get_custom_sort(cls, index_id, sort_type):
"""Get custom sort."""
if sort_type == "asc":
factor_obj = Indexes.get_item_sort(index_id)
script_str = {
"_script": {
"script":
"factor.get(doc[\"control_number\"].value)&&factor.get(doc[\"control_number\"].value) !=0 ? factor.get(doc[\"control_number\"].value):Integer.MAX_VALUE",
"type": "number",
"params": {
"factor": factor_obj
},
"order": "asc"
}
}
default_sort = {'_created': {'order': 'desc'}}
else:
factor_obj = Indexes.get_item_sort(index_id)
script_str = {
"_script": {
"script":
"factor.get(doc[\"control_number\"].value)&&factor.get(doc[\"control_number\"].value) !=0 ? factor.get(doc[\"control_number\"].value):0",
"type": "number",
"params": {
"factor": factor_obj
},
"order": "desc"
}
}
default_sort = {'_created': {'order': 'asc'}}
return script_str, default_sort
def save_sort(self):
"""Save custom sort."""
try:
data = request.get_json()
index_id = data.get("q_id")
sort_data = data.get("sort")
# save data to DB
item_sort = {}
for sort in sort_data:
item_sort[sort.get('id')] = sort.get('custom_sort').get(
index_id)
Indexes.set_item_sort_custom(index_id, item_sort)
# update es
fp = Indexes.get_self_path(index_id)
Indexes.update_item_sort_custom_es(fp.path, sort_data)
jfy = {}
jfy['status'] = 200
jfy['message'] = 'Data is successfully updated.'
return make_response(jsonify(jfy), jfy['status'])
except Exception as ex:
jfy['status'] = 405
jfy['message'] = 'Error'
return make_response(jsonify(jfy), jfy['status'])
def get_permission_filter(comm_id=None):
"""Get permission filter."""
# check permission
is_perm = search_permission.can()
match = Q('match', publish_status='0')
version = Q('match', relation_version_is_last='true')
rng = Q('range', **{'publish_date': {'lte': 'now/d'}})
term_list = []
mst = []
is_perm_paths = Indexes.get_browsing_tree_paths()
search_type = request.values.get('search_type')
if comm_id:
if search_type == config.WEKO_SEARCH_TYPE_DICT['FULL_TEXT']:
self_path = Indexes.get_self_path(comm_id)
if self_path and self_path.path in is_perm_paths:
term_list.append(self_path.path)
path = term_list[0] + '*'
should_path = []
wildcard_path = Q("wildcard", path=path)
should_path.append(wildcard_path)
mst.append(match)
mst.append(rng)
terms = Q('bool', should=should_path)
else: # In case search_type is keyword or index
self_path = Indexes.get_self_path(comm_id)
if self_path and self_path.path in is_perm_paths:
term_list.append(self_path.path)
mst.append(match)
mst.append(rng)
terms = Q('terms', path=term_list)
else:
mst.append(match)
mst.append(rng)
terms = Q('terms', path=is_perm_paths)
mut = []
if is_perm:
user_id, result = check_admin_user()
if result:
shuld = [
Q('match', weko_creator_id=user_id),
Q('match', weko_shared_id=user_id)
]
shuld.append(Q('bool', must=mst))
mut.append(Q('bool', should=shuld, must=[terms]))
mut.append(Q('bool', must=version))
else:
mut = mst
mut.append(terms)
base_mut = [match, version]
mut.append(Q('bool', must=base_mut))
return mut
def get_custom_sort(cls, index_id, sort_type):
"""Get custom sort."""
if sort_type == "asc":
factor_obj = Indexes.get_item_sort(index_id)
script_str = {
"_script": {
"script": {
"source":
"if(params.factor.get(doc[\"control_number\"].value.toString())!=null){params.factor.get(doc[\"control_number\"].value.toString())}else{Integer.MAX_VALUE}",
"lang": "painless",
"params": {
"factor": factor_obj
}
},
"type": "number",
"order": "asc"
}
}
default_sort = {
'_created': {
'order': 'desc',
'unmapped_type': 'long'
}
}
else:
factor_obj = Indexes.get_item_sort(index_id)
script_str = {
"_script": {
"script": {
"source":
"if(params.factor.get(doc[\"control_number\"].value.toString())!=null){params.factor.get(doc[\"control_number\"].value.toString())}else{0}",
"lang": "painless",
"params": {
"factor": factor_obj
}
},
"type": "number",
"order": "desc"
}
}
default_sort = {
'_created': {
'order': 'asc',
'unmapped_type': 'long'
}
}
return script_str, default_sort
def get_permission_filter(comm_id=None):
# check permission
is_perm = search_permission.can()
mut = []
match = Q('match', publish_status='0')
# ava = [Q('range', **{'date.value': {'lte': 'now/d'}}),
# Q('term', **{'date.dateType': 'Available'})]
# rng = Q('nested', path='date', query=Q('bool', must=ava))
ava = Q('range', **{'publish_date': {'lte': 'now/d'}})
rng = ava
mst = []
if comm_id is not None:
path_list = Indexes.get_all_path_list(comm_id)
match_list = []
for p_l in path_list:
match_q = Q('match', path=p_l)
match_list.append(match_q)
mst.append(Q('bool', should=match_list))
if not is_perm:
mut.append(match)
mut.append(rng)
mut.append(get_index_filter()[0])
else:
user_id, result = check_admin_user()
if result:
shuld = [Q('match', weko_creator_id=user_id)]
mut2 = [match, rng]
shuld.append(Q('bool', must=mut2))
if comm_id is not None:
mut.append(Q('bool', should=shuld, must=mst))
else:
mut.append(Q('bool', should=shuld, must=get_index_filter()))
return mut
def get_record_with_hps(cls, uuid):
record = cls.get_record(id_=uuid)
path = []
path.extend(record.get('path'))
harvest_public_state = True
if path:
harvest_public_state = Indexes.get_harvest_public_state(path)
return harvest_public_state, record
def index(self):
"""Bulk delete items and index trees."""
if request.method == 'PUT':
# Do delete items inside the current index tree (maybe root tree)
q = request.values.get('q')
if q is not None and q.isdigit():
current_tree = Indexes.get_index(q)
recursive_tree = Indexes.get_recursive_tree(q)
if current_tree is not None:
# Delete items in current_tree
delete_records(current_tree.id)
# If recursively, then delete all child index trees
# and theirs items
if request.values.get('recursively') == 'true'\
and recursive_tree is not None:
# Delete recursively
direct_child_trees = []
for index, obj in enumerate(recursive_tree):
if obj[1] != current_tree.id:
child_tree = Indexes.get_index(obj[1])
# Do delete items in child_tree
delete_records(child_tree.id)
# Add the level 1 child into the current_tree
if obj[0] == current_tree.id:
direct_child_trees.append(child_tree.id)
# Then do delete child_tree inside current_tree
for cid in direct_child_trees:
# Delete this tree and children
Indexes.delete(cid)
return jsonify({'status': 1})
else:
return jsonify({'status': 0, 'msg': 'Invalid tree'})
"""Render view."""
detail_condition = get_search_detail_keyword('')
return self.render(
current_app.config['WEKO_THEME_ADMIN_ITEM_MANAGEMENT_TEMPLATE'],
management_type='delete',
detail_condition=detail_condition
)
def get_record_cvs(cls, uuid):
"""Get record cvs."""
record = cls.get_record(id_=uuid)
path = []
path.extend(record.get('path'))
coverpage_state = False
if path:
coverpage_state = Indexes.get_coverpage_state(path)
return coverpage_state
def get_index_filter():
paths = Indexes.get_browsing_tree_paths()
mst = []
q_list = []
for path in paths:
match_q = Q('match', path=path)
q_list.append(match_q)
mst.append(Q('bool', should=q_list))
return mst
def get_permission_filter(comm_id=None):
"""Get permission filter."""
# check permission
is_perm = search_permission.can()
match = Q('match', publish_status='0')
version = Q('match', relation_version_is_last='true')
rng = Q('range', **{'publish_date': {'lte': 'now/d'}})
term_list = []
mst = []
is_perm_paths = Indexes.get_browsing_tree_paths()
if comm_id is not None:
self_path = Indexes.get_self_path(comm_id)
if self_path.path in is_perm_paths:
term_list.append(self_path.path)
mst.append(match)
mst.append(rng)
terms = Q('terms', path=term_list)
else:
mst.append(match)
mst.append(rng)
terms = Q('terms', path=is_perm_paths)
mut = []
if is_perm:
user_id, result = check_admin_user()
if result:
shuld = [
Q('match', weko_creator_id=user_id),
Q('match', weko_shared_id=user_id)
]
shuld.append(Q('bool', must=mst))
mut.append(Q('bool', should=shuld, must=[terms]))
mut.append(Q('bool', must=version))
else:
mut = mst
mut.append(terms)
base_mut = [match, version]
mut.append(Q('bool', must=base_mut))
return mut
def convert_item_metadata(self, index_obj):
"""
1. Convert Item Metadata
2. Inject index tree id to dict
3. Set Publish Status
:param index_obj:
:return: dc
"""
# if this item has been deleted
self.delete_es_index_attempt(self.pid)
try:
actions = index_obj.get('actions', 'private')
datastore = RedisStore(redis.StrictRedis.from_url(
current_app.config['CACHE_REDIS_URL']))
cache_key = current_app.config[
'WEKO_DEPOSIT_ITEMS_CACHE_PREFIX'].format(
pid_value=self.pid.pid_value)
data_str = datastore.get(cache_key)
datastore.delete(cache_key)
data = json.loads(data_str)
except:
abort(500, 'Failed to register item')
# Get index path
index_lst = index_obj.get('index', [])
plst = Indexes.get_path_list(index_lst)
if not plst or len(index_lst) != len(plst):
raise PIDResolveRESTError(description='Any tree index has been deleted')
index_lst.clear()
for lst in plst:
index_lst.append(lst.path)
# convert item meta data
dc, jrc, is_edit = json_loader(data, self.pid)
self.data = data
self.jrc = jrc
self.is_edit = is_edit
# Save Index Path on ES
jrc.update(dict(path=index_lst))
dc.update(dict(path=index_lst))
pubs = '1' if 'private' in actions else '0'
ps = dict(publish_status=pubs)
jrc.update(ps)
dc.update(ps)
return dc
def save_sort(self):
"""Save custom sort."""
try:
data = request.get_json()
index_id = data.get("q_id")
sort_data = data.get("sort")
# save data to DB
item_sort = {}
for sort in sort_data:
sd = sort.get('custom_sort').get(index_id)
if sd:
item_sort[sort.get('id')] = sd
Indexes.set_item_sort_custom(index_id, item_sort)
# update es
# fp = Indexes.get_self_path(index_id)
# Indexes.update_item_sort_custom_es(fp.path, sort_data)
jfy = {'status': 200, 'message': 'Data is successfully updated.'}
except Exception:
jfy = {'status': 405, 'message': 'Error.'}
return make_response(jsonify(jfy), jfy['status'])
def handle_replace_new_index() -> list:
"""Validation importing zip file.
:argument
:return
return -- index id import item
"""
from datetime import datetime
now = datetime.now()
index_import = Indexes.get_index_by_name("Index_import")
if index_import:
return [index_import.id]
else:
create_index = Indexes.create(
pid=0,
indexes={'id': int(datetime.timestamp(now) * 10 ** 3),
'value': 'Index_import'}
)
if create_index:
index_import = Indexes.get_index_by_name("Index_import")
if index_import:
return [index_import.id]
return []
def handle_check_index(list_index: list) -> bool:
"""Handle check index.
:argument
list_index -- {list} list index id.
:return
return -- true if exist.
"""
result = True
index_lst = []
if list_index:
index_id_lst = []
for index in list_index:
indexes = str(index).split('/')
index_id_lst.append(indexes[len(indexes) - 1])
index_lst = index_id_lst
plst = Indexes.get_path_list(index_lst)
if not plst or len(index_lst) != len(plst):
result = False
return result
def is_private_index(record):
"""Check index of workflow is private."""
return not Indexes.is_public_state(copy.deepcopy(record.get("path")))
def _get_index_search_query():
"""Get index search query."""
query_q = {
"from": 0,
"size": 10000,
"_source": {
"excludes": ["content", "_item_metadata"]
},
"query": {
"bool": {
"must": [{
"match": {
"path.tree": "@index"
}
}, {
"match": {
"relation_version_is_last": "true"
}
}]
}
},
"post_filter": {
"bool": {
"must": [{
"match": {
"publish_status": "0"
}
}, {
"range": {
"publish_date": {
"lte": "now/d"
}
}
}]
}
}
}
q = str(index_id)
if q != str(current_app.config.get("WEKO_ROOT_INDEX",
WEKO_ROOT_INDEX)):
post_filter = query_q['post_filter']
if post_filter:
list_path = Indexes.get_list_path_publish(index_id)
post_filter['bool']['must'].append(
{"terms": {
"path": list_path
}})
# create search query
try:
fp = Indexes.get_self_path(q)
query_q = json.dumps(query_q).replace("@index", fp.path)
query_q = json.loads(query_q)
except BaseException:
pass
else:
post_filter = query_q['post_filter']
if post_filter:
list_path = Indexes.get_list_path_publish(index_id)
post_filter['bool']['must'].append(
{"terms": {
"path": list_path
}})
wild_card = []
child_list = Indexes.get_child_list(q)
if child_list:
for item in child_list:
wc = {"wildcard": {"path.tree": item.cid}}
wild_card.append(wc)
query_q['query']['bool']['must'] = [{
"bool": {
"should": wild_card
}
}, {
"match": {
"relation_version_is_last": "true"
}
}]
return query_q
def get(self, **kwargs):
"""Search records.
:returns: the search result containing hits and aggregations as
returned by invenio-search.
"""
page = request.values.get('page', 1, type=int)
size = request.values.get('size', 20, type=int)
community_id = request.values.get('community')
params = {}
if current_app.config['RECORDS_REST_FACETS'] and \
current_app.config['SEARCH_UI_SEARCH_INDEX'] and \
'post_filters' in current_app.config[
'RECORDS_REST_FACETS'
][current_app.config[
'SEARCH_UI_SEARCH_INDEX'
]]:
post_filters = current_app.config['RECORDS_REST_FACETS'][
current_app.config['SEARCH_UI_SEARCH_INDEX']]['post_filters']
for param in post_filters:
value = request.args.getlist(param)
if value:
params[param] = value
if page * size >= self.max_result_window:
raise MaxResultWindowRESTError()
urlkwargs = dict()
search_obj = self.search_class()
search = search_obj.with_preference_param().params(version=True)
search = search[(page - 1) * size:page * size]
search, qs_kwargs = self.search_factory(self, search)
query = request.values.get('q')
if query:
urlkwargs['q'] = query
# Execute search
for param in params:
query_key = current_app.config['WEKO_FACETED_SEARCH_MAPPING'][
param]
search = search.post_filter({'terms': {query_key: params[param]}})
search_result = search.execute()
# Generate links for prev/next
urlkwargs.update(
size=size,
_external=True,
)
# endpoint = '.{0}_index'.format(
# current_records_rest.default_endpoint_prefixes[self.pid_type])
links = dict(self=url_for(
'weko_search_rest.recid_index', page=page, **urlkwargs))
if page > 1:
links['prev'] = url_for('weko_search_rest.recid_index',
page=page - 1,
**urlkwargs)
if size * page < search_result.hits.total and \
size * page < self.max_result_window:
links['next'] = url_for('weko_search_rest.recid_index',
page=page + 1,
**urlkwargs)
# aggs result identify
rd = search_result.to_dict()
q = request.values.get('q') or '0'
lang = current_i18n.language
try:
paths = Indexes.get_self_list(q, community_id)
except BaseException:
paths = []
agp = rd["aggregations"]["path"]["buckets"]
nlst = []
for p in paths:
m = 0
for k in range(len(agp)):
if p.path == agp[k].get("key"):
agp[k]["name"] = p.name if lang == "ja" else p.name_en
date_range = agp[k].pop("date_range")
no_available = agp[k].pop("no_available")
pub = dict()
bkt = date_range['available']['buckets']
if bkt:
for d in bkt:
pub["pub_cnt" if d.
get("to") else "un_pub_cnt"] = d.get(
"doc_count")
pub["un_pub_cnt"] += no_available['doc_count']
agp[k]["date_range"] = pub
comment = p.comment
agp[k]["comment"] = comment,
result = agp.pop(k)
result["comment"] = comment
nlst.append(result)
m = 1
break
if m == 0:
index_id = p.path if '/' not in p.path \
else p.path.split('/').pop()
index_info = Indexes.get_index(index_id=index_id)
rss_status = index_info.rss_status
nd = {
'doc_count': 0,
'key': p.path,
'name': p.name if lang == "ja" else p.name_en,
'date_range': {
'pub_cnt': 0,
'un_pub_cnt': 0
},
'rss_status': rss_status,
'comment': p.comment,
}
nlst.append(nd)
agp.clear()
# process index tree image info
if len(nlst):
index_id = nlst[0].get('key')
index_id = index_id if '/' not in index_id \
else index_id.split('/').pop()
index_info = Indexes.get_index(index_id=index_id)
# update by weko_dev17 at 2019/04/04
if len(index_info.image_name) > 0:
nlst[0]['img'] = index_info.image_name
nlst[0]['display_format'] = index_info.display_format
nlst[0]['rss_status'] = index_info.rss_status
# Update rss_status for index child
for idx in range(0, len(nlst)):
index_id = nlst[idx].get('key')
index_id = index_id if '/' not in index_id \
else index_id.split('/').pop()
index_info = Indexes.get_index(index_id=index_id)
nlst[idx]['rss_status'] = index_info.rss_status
agp.append(nlst)
for hit in rd['hits']['hits']:
try:
# Register comment
_comment = list()
_comment.append(hit['_source']['title'][0])
hit['_source']['_comment'] = _comment
# Register custom_sort
cn = hit['_source']['control_number']
if index_info.item_custom_sort.get(cn):
hit['_source']['custom_sort'] = {
str(index_info.id):
str(index_info.item_custom_sort.get(cn))
}
except Exception:
pass
# add info (headings & page info)
try:
item_type_list = {}
for hit in rd['hits']['hits']:
# get item type schema
item_type_id = \
hit['_source']['_item_metadata']['item_type_id']
if item_type_id in item_type_list:
item_type = copy.deepcopy(item_type_list[item_type_id])
else:
item_type = ItemType.query.filter_by(
id=item_type_id).first()
item_type_list[item_type_id] = copy.deepcopy(item_type)
# heading
heading = get_heading_info(hit, lang, item_type)
hit['_source']['heading'] = heading
# page info
if 'pageStart' not in hit['_source']:
hit['_source']['pageStart'] = []
if 'pageEnd' not in hit['_source']:
hit['_source']['pageEnd'] = []
except Exception as ex:
current_app.logger.error(ex)
return self.make_response(
pid_fetcher=self.pid_fetcher,
search_result=rd,
links=links,
item_links_factory=self.links_factory,
)
def create(cls, journals=None):
"""
Create the journals. Delete all journals before creation.
:param journals: the journal information (dictinary).
:returns: The :class:`Journal` instance lists or None.
"""
def _add_journal(data):
with db.session.begin_nested():
journal = Journal(**data)
db.session.add(journal)
db.session.commit()
if not isinstance(journals, dict):
return
data = dict()
is_ok = True
try:
cid = journals.get('id')
if not cid:
return
data["id"] = cid
# check index id.
index_id = journals.get('index_id')
if not index_id:
return
index_info = Indexes.get_index(index_id=index_id, with_count=True)
if index_info:
data["index_id"] = index_id
else:
return
data["publication_title"] = journals.get('publication_title')
data["print_identifier"] = journals.get('print_identifier')
data["online_identifier"] = journals.get('online_identifier')
data["date_first_issue_online"] = journals.get(
'date_first_issue_online')
data["num_first_vol_online"] = journals.get('num_first_vol_online')
data["num_first_issue_online"] = journals.get(
'num_first_issue_online')
data["date_last_issue_online"] = journals.get(
'date_last_issue_online')
data["num_last_vol_online"] = journals.get('num_last_vol_online')
data["num_last_issue_online"] = journals.get(
'num_last_issue_online')
data["embargo_info"] = journals.get('embargo_info')
data["coverage_depth"] = journals.get('coverage_depth')
data["coverage_notes"] = journals.get('coverage_notes')
data["publisher_name"] = journals.get('publisher_name')
data["publication_type"] = journals.get('publication_type')
data["parent_publication_title_id"] = journals.get(
'parent_publication_title_id')
data["preceding_publication_title_id"] = journals.get(
'preceding_publication_title_id')
data["access_type"] = journals.get('access_type')
data["language"] = journals.get('language')
data["title_alternative"] = journals.get('title_alternative')
data["title_transcription"] = journals.get('title_transcription')
data["ncid"] = journals.get('ncid')
data["ndl_callno"] = journals.get('ndl_callno')
data["ndl_bibid"] = journals.get('ndl_bibid')
data["jstage_code"] = journals.get('jstage_code')
data["ichushi_code"] = journals.get('ichushi_code')
data["is_output"] = journals.get('is_output')
# get current user logged id.
data["owner_user_id"] = current_user.get_id()
_add_journal(data)
except IntegrityError as ie:
is_ok = False
current_app.logger.debug(ie)
except Exception as ex:
is_ok = False
current_app.logger.debug(ex)
finally:
del data
if not is_ok:
db.session.rollback()
return is_ok
def navi(self):
"""Return the path name."""
return Indexes.get_path_name(self.get('path', []))
def convert_item_metadata(self, index_obj, data=None):
"""Convert Item Metadat.
1. Convert Item Metadata
2. Inject index tree id to dict
3. Set Publish Status
:param index_obj:
:return: dc
"""
# if this item has been deleted
self.delete_es_index_attempt(self.pid)
try:
actions = index_obj.get('actions', 'private')
if not data:
datastore = RedisStore(
redis.StrictRedis.from_url(
current_app.config['CACHE_REDIS_URL']))
cache_key = current_app.config[
'WEKO_DEPOSIT_ITEMS_CACHE_PREFIX'].format(
pid_value=self.pid.pid_value)
data_str = datastore.get(cache_key)
datastore.delete(cache_key)
data = json.loads(data_str.decode('utf-8'))
except BaseException:
abort(500, 'Failed to register item')
# Get index path
index_lst = index_obj.get('index', [])
# Prepare index id list if the current index_lst is a path list
if index_lst:
index_id_lst = []
for index in index_lst:
indexes = str(index).split('/')
index_id_lst.append(indexes[len(indexes) - 1])
index_lst = index_id_lst
plst = Indexes.get_path_list(index_lst)
if not plst or len(index_lst) != len(plst):
raise PIDResolveRESTError(
description='Any tree index has been deleted')
index_lst.clear()
for lst in plst:
index_lst.append(lst.path)
# convert item meta data
dc, jrc, is_edit = json_loader(data, self.pid)
self.data = data
self.jrc = jrc
self.is_edit = is_edit
# Save Index Path on ES
jrc.update(dict(path=index_lst))
# add at 20181121 start
sub_sort = {}
for pth in index_lst:
# es setting
sub_sort[pth[-13:]] = ""
jrc.update(dict(custom_sort=sub_sort))
dc.update(dict(custom_sort=sub_sort))
dc.update(dict(path=index_lst))
pubs = '1' if 'private' in actions else '0'
ps = dict(publish_status=pubs)
jrc.update(ps)
dc.update(ps)
return dc
def _get_index_earch_query():
query_q = {
"_source": {
"excludes": ['content']
},
"query": {
"bool": {
"must": [{
"match": {
"path.tree": "@index"
}
}, {
"match": {
"relation_version_is_last": "true"
}
}]
}
},
"aggs": {
"path": {
"terms": {
"field": "path.tree",
"include": "@index|@index/[^/]+",
"size": "@count"
},
"aggs": {
"date_range": {
"filter": {
"match": {
"publish_status": "0"
}
},
"aggs": {
"available": {
"range": {
"field":
"publish_date",
"ranges": [{
"from": "now+1d/d"
}, {
"to": "now+1d/d"
}]
},
}
}
},
"no_available": {
"filter": {
"bool": {
"must_not": [{
"match": {
"publish_status": "0"
}
}]
}
}
}
}
}
},
"post_filter": {}
}
# add item type aggs
query_q['aggs']['path']['aggs']. \
update(get_item_type_aggs(search._index[0]))
q = request.values.get('q') if index_id is None else index_id
if q:
mut = get_permission_filter(q)
else:
mut = get_permission_filter()
if mut:
mut = list(map(lambda x: x.to_dict(), mut))
post_filter = query_q['post_filter']
if mut[0].get('bool'):
post_filter['bool'] = mut[0]['bool']
else:
post_filter['bool'] = {'must': mut}
# create search query
if q:
try:
fp = Indexes.get_self_path(q)
if fp:
query_q = json.dumps(query_q).replace("@index", fp.path)
query_q = json.loads(query_q)
except BaseException:
pass
query_q = json.dumps(query_q).replace("@count",
str(Indexes.get_index_count()))
query_q = json.loads(query_q)
return query_q
def build_rss_xml(data=None, index_id=0, page=1, count=20, term=0, lang=''):
"""Build RSS data as XML format.
Arguments:
data {dictionary} -- Elastic search data
term {int} -- The term
Returns:
xml response -- RSS data as XML
"""
root_url = str(request.url_root).replace('/api/', '/')
root = Et.Element('rdf:RDF')
root.set('xmlns', config.WEKO_XMLNS)
root.set('xmlns:rdf', config.WEKO_XMLNS_RDF)
root.set('xmlns:rdfs', config.WEKO_XMLNS_RDFS)
root.set('xmlns:dc', config.WEKO_XMLNS_DC)
root.set('xmlns:prism', config.WEKO_XMLNS_PRISM)
root.set('xmlns:lang', lang)
# First layer
requested_url = root_url + 'rss/records?index_id=' + str(index_id) + \
'&page=' + str(page) + '&term=' + str(term) + \
'&count=' + str(count) + '&lang=' + str(lang)
channel = Et.SubElement(root, 'channel')
channel.set('rdf:about', requested_url)
# Channel layer
Et.SubElement(channel, 'title').text = 'WEKO3'
Et.SubElement(channel, 'link').text = requested_url
if index_id:
index_detail = Indexes.get_index(index_id)
Et.SubElement(channel, 'description').text = index_detail.comment \
or index_detail.index_name \
or index_detail.index_name_english
else:
Et.SubElement(channel, 'description').text = \
theme_config.THEME_SITENAME
current_time = datetime.now()
Et.SubElement(channel,
'dc:date').text = current_time.isoformat() + '+00:00'
items = Et.SubElement(channel, 'items')
seq = Et.SubElement(items, 'rdf:Seq')
if not data or not isinstance(data, list):
xml_str = tostring(root, encoding='utf-8')
xml_str = str.encode(config.WEKO_XML_FORMAT) + xml_str
return Response(xml_str, mimetype='text/xml')
items = [idx for idx in range((page - 1) * count, page * count)]
item_idx = 0
# add item layer
for data_item in data:
if item_idx not in items:
item_idx = item_idx + 1
continue
item = Et.Element('item')
item.set('rdf:about', find_rss_value(data_item, 'link'))
Et.SubElement(item, 'title').text = find_rss_value(data_item, 'title')
Et.SubElement(item, 'link').text = find_rss_value(data_item, 'link')
see_also = Et.SubElement(item, 'rdfs:seeAlso')
see_also.set('rdf:resource', find_rss_value(data_item, 'seeAlso'))
if isinstance(find_rss_value(data_item, 'creator'), list):
for creator in find_rss_value(data_item, 'creator'):
Et.SubElement(item, 'dc:creator').text = creator
else:
Et.SubElement(item, 'dc:creator').text = find_rss_value(
data_item, 'creator')
Et.SubElement(item, 'dc:publisher').text = find_rss_value(
data_item, 'publisher')
Et.SubElement(item, 'prism:publicationName').text = find_rss_value(
data_item, 'sourceTitle')
Et.SubElement(item,
'prism:issn').text = find_rss_value(data_item, 'issn')
Et.SubElement(item, 'prism:volume').text = find_rss_value(
data_item, 'volume')
Et.SubElement(item, 'prism:number').text = find_rss_value(
data_item, 'issue')
Et.SubElement(item, 'prism:startingPage').text = find_rss_value(
data_item, 'pageStart')
Et.SubElement(item, 'prism:endingPage').text = find_rss_value(
data_item, 'pageEnd')
Et.SubElement(item, 'prism:publicationDate').text = find_rss_value(
data_item, 'date')
Et.SubElement(item, 'description').text = find_rss_value(
data_item, 'description')
Et.SubElement(item,
'dc:date').text = find_rss_value(data_item, '_updated')
li = Et.SubElement(seq, 'rdf:li')
li.set('rdf:resource', find_rss_value(data_item, 'link'))
root.append(item)
item_idx = item_idx + 1
xml_str = tostring(root, encoding='utf-8')
xml_str = str.encode(config.WEKO_XML_FORMAT) + xml_str
response = current_app.response_class()
response.data = xml_str
response.headers['Content-Type'] = 'application/xml'
return response
def get_records(**kwargs):
"""Get records paginated."""
def index_ids_has_future_date():
"""Get indexes."""
query = Index.query.filter(
Index.public_state.is_(True),
Index.public_date > datetime.now(),
Index.harvest_public_state.is_(True)
)
indexes = query.all() or []
index_ids = [index.id for index in indexes]
return index_ids
def get_records_has_doi():
"""Get object_uuid of PersistentIdentifier."""
# Get object_uuid of PersistentIdentifier
query = PersistentIdentifier.query.filter(
PersistentIdentifier.pid_type == 'doi'
)
pids = query.all() or []
object_uuids = [pid.object_uuid for pid in pids]
# Get RecordMetadata
query = RecordMetadata.query.filter(
RecordMetadata.id.in_(object_uuids)
)
records = query.all() or []
return records
def add_condition_doi_and_future_date(query):
"""Add condition which do not get DOI."""
index_ids = index_ids_has_future_date()
records = get_records_has_doi()
for record in records:
paths = record.json.get('path', [])
for path in paths:
if path in index_ids:
query = query.post_filter(
'bool',
**{'must_not': [
{'term': {'_id': str(record.id)}}]})
continue
from weko_index_tree.api import Indexes
page_ = kwargs.get('resumptionToken', {}).get('page', 1)
size_ = current_app.config['OAISERVER_PAGE_SIZE']
scroll = current_app.config['OAISERVER_RESUMPTION_TOKEN_EXPIRE_TIME']
scroll_id = kwargs.get('resumptionToken', {}).get('scroll_id')
if not scroll_id:
search = OAIServerSearch(
index=current_app.config['INDEXER_DEFAULT_INDEX'],
).params(
scroll='{0}s'.format(scroll),
).extra(
version='true',
).sort(
{'control_number': {'order': 'asc'}}
)[(page_ - 1) * size_:page_ * size_]
if 'set' in kwargs:
search = search.query('match', **{'_oai.sets': kwargs['set']})
time_range = {}
if 'from_' in kwargs:
time_range['gte'] = kwargs['from_']
if 'until' in kwargs:
time_range['lte'] = kwargs['until']
if time_range:
search = search.filter('range', **{'_updated': time_range})
search = search.query('match', **{'relation_version_is_last': 'true'})
index_paths = Indexes.get_harverted_index_list()
query_filter = [
# script get deleted items.
{"bool": {"must_not": {"exists": {"field": "path"}}}}
]
for index_path in index_paths:
query_filter.append({
"wildcard": {
"path": index_path
}
})
search = search.query(
'bool', **{'must': [{'bool': {'should': query_filter}}]})
add_condition_doi_and_future_date(search)
response = search.execute().to_dict()
else:
response = current_search_client.scroll(
scroll_id=scroll_id,
scroll='{0}s'.format(scroll),
)
class Pagination(object):
"""Dummy pagination class."""
page = page_
per_page = size_
def __init__(self, response):
"""Initilize pagination."""
self.response = response
self.total = response['hits']['total']
self._scroll_id = response.get('_scroll_id')
# clean descriptor on last page
if not self.has_next:
current_search_client.clear_scroll(
scroll_id=self._scroll_id
)
self._scroll_id = None
@cached_property
def has_next(self):
"""Return True if there is next page."""
return self.page * self.per_page <= self.total
@cached_property
def next_num(self):
"""Return next page number."""
return self.page + 1 if self.has_next else None
@property
def items(self):
"""Return iterator."""
from datetime import datetime
for result in self.response['hits']['hits']:
if '_oai' in result['_source']:
yield {
'id': result['_id'],
'json': result,
'updated': datetime.strptime(
result['_source']['_updated'][:19],
'%Y-%m-%dT%H:%M:%S'
),
}
return Pagination(response)
def search():
"""Index Search page ui."""
search_type = request.args.get('search_type',
WEKO_SEARCH_TYPE_DICT['FULL_TEXT'])
get_args = request.args
community_id = ""
ctx = {'community': None}
cur_index_id = search_type if search_type not in \
(WEKO_SEARCH_TYPE_DICT['FULL_TEXT'], WEKO_SEARCH_TYPE_DICT[
'KEYWORD'], ) else None
if 'community' in get_args:
from weko_workflow.api import GetCommunity
comm = GetCommunity.get_community_by_id(request.args.get('community'))
ctx = {'community': comm}
community_id = comm.id
# Get the design for widget rendering
page, render_widgets = get_design_layout(
community_id or current_app.config['WEKO_THEME_DEFAULT_COMMUNITY'])
# Get index style
style = IndexStyle.get(
current_app.config['WEKO_INDEX_TREE_STYLE_OPTIONS']['id'])
width = style.width if style else '3'
# add at 1206 for search management
sort_options, display_number = SearchSetting.get_results_setting()
ts = time.time()
disply_setting = dict(size=display_number, timestamp=ts)
detail_condition = get_search_detail_keyword('')
export_settings = AdminSettings.get('item_export_settings') or \
AdminSettings.Dict2Obj(
current_app.config['WEKO_ADMIN_DEFAULT_ITEM_EXPORT_SETTINGS'])
height = style.height if style else None
if 'item_link' in get_args:
from weko_workflow.api import WorkActivity
activity_id = request.args.get('item_link')
workflow_activity = WorkActivity()
activity_detail, item, steps, action_id, cur_step, temporary_comment,\
approval_record, step_item_login_url, histories, res_check, pid, \
community_id, ctx = workflow_activity.get_activity_index_search(
activity_id=activity_id)
# Get ex-Item Links
recid = item['pid'].get('value') if item.get('pid') else None
if recid:
pid_without_ver = recid.split('.')[0]
item_link = ItemLink.get_item_link_info(pid_without_ver)
ctx['item_link'] = item_link
return render_template(
'weko_workflow/activity_detail.html',
page=page,
render_widgets=render_widgets,
activity=activity_detail,
item=item,
steps=steps,
action_id=action_id,
cur_step=cur_step,
temporary_comment=temporary_comment,
record=approval_record,
step_item_login_url=step_item_login_url,
histories=histories,
res_check=res_check,
pid=pid,
index_id=cur_index_id,
community_id=community_id,
width=width,
height=height,
allow_item_exporting=export_settings.allow_item_exporting,
is_permission=check_permission(),
is_login=bool(current_user.get_id()),
**ctx)
else:
journal_info = None
index_display_format = '1'
check_site_license_permission()
send_info = dict()
send_info['site_license_flag'] = True \
if hasattr(current_user, 'site_license_flag') else False
send_info['site_license_name'] = current_user.site_license_name \
if hasattr(current_user, 'site_license_name') else ''
if search_type in WEKO_SEARCH_TYPE_DICT.values():
searched.send(current_app._get_current_object(),
search_args=get_args,
info=send_info)
if search_type == WEKO_SEARCH_TYPE_DICT['INDEX']:
cur_index_id = request.args.get('q', '0')
journal_info = get_journal_info(cur_index_id)
index_info = Indexes.get_index(cur_index_id)
if index_info:
index_display_format = index_info.display_format
if index_display_format == '2':
disply_setting = dict(size=100, timestamp=ts)
if hasattr(current_i18n, 'language'):
index_link_list = get_index_link_list(current_i18n.language)
else:
index_link_list = get_index_link_list()
return render_template(
current_app.config['SEARCH_UI_SEARCH_TEMPLATE'],
page=page,
render_widgets=render_widgets,
index_id=cur_index_id,
community_id=community_id,
sort_option=sort_options,
disply_setting=disply_setting,
detail_condition=detail_condition,
width=width,
height=height,
index_link_enabled=style.index_link_enabled,
index_link_list=index_link_list,
journal_info=journal_info,
index_display_format=index_display_format,
allow_item_exporting=export_settings.allow_item_exporting,
is_permission=check_permission(),
is_login=bool(current_user.get_id()),
**ctx)
def get_child_list(index_id=0):
"""Get child id list to index list display."""
return jsonify(Indexes.get_child_id_list(index_id))
def index(self):
"""Index Search page ui."""
search_type = request.args.get('search_type', '0')
getArgs = request.args
community_id = ""
ctx = {'community': None}
cur_index_id = search_type if search_type not in (
'0',
'1',
) else None
if 'community' in getArgs:
from weko_workflow.api import GetCommunity
comm = GetCommunity.get_community_by_id(
request.args.get('community'))
ctx = {'community': comm}
community_id = comm.id
# Get index style
style = IndexStyle.get(
current_app.config['WEKO_INDEX_TREE_STYLE_OPTIONS']['id'])
width = style.width if style else '3'
detail_condition = get_search_detail_keyword('')
height = style.height if style else None
if 'item_management' in getArgs:
management_type = request.args.get('item_management', 'sort')
has_items = False
has_child_trees = False
if management_type == 'delete':
# Does this tree has items or children?
q = request.args.get('q')
if q is not None and q.isdigit():
current_tree = Indexes.get_index(q)
recursive_tree = Indexes.get_recursive_tree(q)
if current_tree is not None:
tree_items = get_tree_items(current_tree.id)
has_items = len(tree_items) > 0
if recursive_tree is not None:
has_child_trees = len(recursive_tree) > 1
return self.render(
current_app.
config['WEKO_THEME_ADMIN_ITEM_MANAGEMENT_TEMPLATE'],
index_id=cur_index_id,
community_id=community_id,
width=width,
height=height,
management_type=management_type,
fields=current_app.config['WEKO_RECORDS_UI_BULK_UPDATE_FIELDS']
['fields'],
licences=current_app.
config['WEKO_RECORDS_UI_BULK_UPDATE_FIELDS']['licences'],
has_items=has_items,
has_child_trees=has_child_trees,
detail_condition=detail_condition,
**ctx)
else:
return abort(500)
def get_index(self):
"""Get Index obj relate to repository_id."""
if self.index_id:
return Indexes.get_index(self.index_id)
else:
return None
def get(self, **kwargs):
"""Search records.
:returns: the search result containing hits and aggregations as
returned by invenio-search.
"""
page = request.values.get('page', 1, type=int)
size = request.values.get('size', 20, type=int)
if page * size >= self.max_result_window:
raise MaxResultWindowRESTError()
urlkwargs = dict()
search_obj = self.search_class()
search = search_obj.with_preference_param().params(version=True)
search = search[(page - 1) * size:page * size]
search, qs_kwargs = self.search_factory(self, search)
urlkwargs.update(qs_kwargs)
# Execute search
search_result = search.execute()
# Generate links for prev/next
urlkwargs.update(
size=size,
_external=True,
)
# endpoint = '.{0}_index'.format(
# current_records_rest.default_endpoint_prefixes[self.pid_type])
links = dict(self=url_for(
'weko_search_rest.recid_index', page=page, **urlkwargs))
if page > 1:
links['prev'] = url_for('weko_search_rest.recid_index',
page=page - 1,
**urlkwargs)
if size * page < search_result.hits.total and \
size * page < self.max_result_window:
links['next'] = url_for('weko_search_rest.recid_index',
page=page + 1,
**urlkwargs)
# aggs result identify
rd = search_result.to_dict()
q = request.values.get('q')
lang = current_i18n.language
if q:
try:
paths = Indexes.get_self_list(q)
except BaseException:
paths = []
agp = rd["aggregations"]["path"]["buckets"]
nlst = []
for p in paths:
m = 0
for k in range(len(agp)):
if p.path == agp[k].get("key"):
agp[k]["name"] = p.name if lang == "ja" else p.name_en
date_range = agp[k].pop("date_range")
no_available = agp[k].pop("no_available")
pub = dict()
bkt = date_range['available']['buckets']
if bkt:
for d in bkt:
pub["pub_cnt" if d.
get("to") else "un_pub_cnt"] = d.get(
"doc_count")
pub["un_pub_cnt"] += no_available['doc_count']
agp[k]["date_range"] = pub
nlst.append(agp.pop(k))
m = 1
break
if m == 0:
nd = {
'doc_count': 0,
'key': p.path,
'name': p.name if lang == "ja" else p.name_en,
'date_range': {
'pub_cnt': 0,
'un_pub_cnt': 0
}
}
nlst.append(nd)
agp.clear()
# process index tree image info
if len(nlst):
index_id = nlst[0].get('key')
index_id = index_id if '/' not in index_id \
else index_id.split('/').pop()
index_info = Indexes.get_index(index_id=index_id)
if index_info.display_format == '2' \
and len(index_info.image_name) > 0:
nlst[0]['img'] = index_info.image_name
agp.append(nlst)
current_app.logger.debug(rd)
return self.make_response(
pid_fetcher=self.pid_fetcher,
search_result=rd,
links=links,
item_links_factory=self.links_factory,
)
def get_index(self):
"""Get Index obj by repository_id."""
if self.repository_id:
return Indexes.get_index(self.repository_id)
else:
return None