def search_share_atom(**kwargs):
q = request.args.get('q', '*')
sort = request.args.get('sort', 'dateUpdated')
# we want the results per page to be constant between pages
# TODO - move this functionality into build_query in util
start = util.compute_start(request.args.get('page', 1), RESULTS_PER_PAGE)
query = build_query(q, size=RESULTS_PER_PAGE, start=start, sort=sort)
try:
search_results = search.search_share(query, index='share_v1')
except MalformedQueryError:
raise HTTPError(http.BAD_REQUEST)
except IndexNotFoundError:
search_results = {'count': 0, 'results': []}
atom_url = api_url_for('search_share_atom', _xml=True, _absolute=True)
return util.create_atom_feed(name='SHARE',
data=search_results['results'],
query=q,
size=RESULTS_PER_PAGE,
start=start,
url=atom_url,
to_atom=share_search.to_atom)
def search_share_atom(**kwargs):
q = request.args.get('q', '*')
sort = request.args.get('sort', 'dateUpdated')
# we want the results per page to be constant between pages
# TODO - move this functionality into build_query in util
start = util.compute_start(request.args.get('page', 1), RESULTS_PER_PAGE)
query = build_query(q, size=RESULTS_PER_PAGE, start=start, sort=sort)
try:
search_results = search.search_share(query)
except MalformedQueryError:
raise HTTPError(http.BAD_REQUEST)
except IndexNotFoundError:
search_results = {
'count': 0,
'results': []
}
atom_url = api_url_for('search_share_atom', _xml=True, _absolute=True)
return util.create_atom_feed(
name='SHARE',
data=search_results['results'],
query=q,
size=RESULTS_PER_PAGE,
start=start,
url=atom_url,
to_atom=share_search.to_atom
)
def search_share():
tick = time.time()
results = {}
count = request.args.get('count') is not None
raw = request.args.get('raw') is not None
version = request.args.get('v')
if version:
index = settings.SHARE_ELASTIC_INDEX_TEMPLATE.format(version)
else:
index = settings.SHARE_ELASTIC_INDEX
if request.method == 'POST':
query = request.get_json()
elif request.method == 'GET':
query = build_query(request.args.get('q', '*'),
request.args.get('from', 0),
request.args.get('size', 10),
sort=request.args.get('sort'))
if count:
results = search.count_share(query, index=index)
else:
results = search.search_share(query, raw, index=index)
results['time'] = round(time.time() - tick, 2)
return results
def search_share():
tick = time.time()
results = {}
count = request.args.get('count') is not None
raw = request.args.get('raw') is not None
version = request.args.get('v')
if version:
index = settings.SHARE_ELASTIC_INDEX_TEMPLATE.format(version)
else:
index = settings.SHARE_ELASTIC_INDEX
if request.method == 'POST':
query = request.get_json()
elif request.method == 'GET':
query = build_query(
request.args.get('q', '*'),
request.args.get('from', 0),
request.args.get('size', 10),
sort=request.args.get('sort')
)
if count:
results = search.count_share(query, index=index)
else:
results = search.search_share(query, raw, index=index)
results['time'] = round(time.time() - tick, 2)
return results
def get_queryset(self):
query = self.request.query_params.get('q', '*')
page = int(self.request.query_params.get('page', '1'))
page_size = min(int(self.request.query_params.get('page[size]', REST_FRAMEWORK['PAGE_SIZE'])), MAX_PAGE_SIZE)
start = (page - 1) * page_size
try:
results = search.search(build_query(query, start=start, size=page_size), doc_type=self.doc_type, raw=True)
except MalformedQueryError as e:
raise ValidationError(e.message)
return results
def get_queryset(self):
query = self.request.query_params.get('q', '*')
page = int(self.request.query_params.get('page', '1'))
page_size = min(int(self.request.query_params.get('page[size]', REST_FRAMEWORK['PAGE_SIZE'])), MAX_PAGE_SIZE)
start = (page - 1) * page_size
try:
results = search.search(build_query(query, start=start, size=page_size), doc_type=self.doc_type, raw=True)
except MalformedQueryError as e:
raise ValidationError(e.message)
return results
def get_queryset(self, query=None):
page = int(self.request.query_params.get('page', '1'))
page_size = min(int(self.request.query_params.get('page[size]', REST_FRAMEWORK['PAGE_SIZE'])), MAX_PAGE_SIZE)
start = (page - 1) * page_size
if query:
# Parser has built query, but needs paging info
query['from'] = start
query['size'] = page_size
else:
query = build_query(self.request.query_params.get('q', '*'), start=start, size=page_size)
try:
results = search.search(query, doc_type=self.doc_type, raw=True)
except MalformedQueryError as e:
raise ValidationError(e.message)
return results
def get_queryset(self, query=None):
page = int(self.request.query_params.get('page', '1'))
page_size = min(int(self.request.query_params.get('page[size]', REST_FRAMEWORK['PAGE_SIZE'])), MAX_PAGE_SIZE)
start = (page - 1) * page_size
if query:
# Parser has built query, but needs paging info
query['from'] = start
query['size'] = page_size
else:
query = build_query(self.request.query_params.get('q', '*'), start=start, size=page_size)
try:
results = search.search(query, doc_type=self.doc_type, raw=True)
except MalformedQueryError as e:
raise ValidationError(e.message)
return results
def search_search(**kwargs):
_type = kwargs.get('type', None)
tick = time.time()
results = {}
if request.method == 'POST':
results = search.search(request.get_json(), doc_type=_type)
elif request.method == 'GET':
q = request.args.get('q', '*')
# TODO Match javascript params?
start = request.args.get('from', '0')
size = request.args.get('size', '10')
results = search.search(build_query(q, start, size), doc_type=_type)
results['time'] = round(time.time() - tick, 2)
return results
def search_search(**kwargs):
_type = kwargs.get('type', None)
tick = time.time()
results = {}
if request.method == 'POST':
results = search.search(request.get_json(), doc_type=_type)
elif request.method == 'GET':
q = request.args.get('q', '*')
# TODO Match javascript params?
start = request.args.get('from', '0')
size = request.args.get('size', '10')
results = search.search(build_query(q, start, size), doc_type=_type)
results['time'] = round(time.time() - tick, 2)
return results
def search_share_atom(**kwargs):
json_query = request.args.get('jsonQuery')
start = util.compute_start(request.args.get('page', 1), RESULTS_PER_PAGE)
if not json_query:
q = request.args.get('q', '*')
sort = request.args.get('sort')
# we want the results per page to be constant between pages
# TODO - move this functionality into build_query in util
query = build_query(q, size=RESULTS_PER_PAGE, start=start, sort=sort)
else:
query = json.loads(unquote(json_query))
query['from'] = start
query['size'] = RESULTS_PER_PAGE
# Aggregations are expensive, and we really don't want to
# execute them if they won't be used
for field in ['aggs', 'aggregations']:
if query.get(field):
del query[field]
q = query # Do we really want to display this?
try:
search_results = search.search_share(query)
except MalformedQueryError:
raise HTTPError(http.BAD_REQUEST)
except IndexNotFoundError:
search_results = {
'count': 0,
'results': []
}
atom_url = api_url_for('search_share_atom', _xml=True, _absolute=True)
return util.create_atom_feed(
name='SHARE',
data=search_results['results'],
query=q,
size=RESULTS_PER_PAGE,
start=start,
url=atom_url,
to_atom=share_search.to_atom
)
def search_share_atom(**kwargs):
json_query = request.args.get('jsonQuery')
start = util.compute_start(request.args.get('page', 1), RESULTS_PER_PAGE)
if not json_query:
q = request.args.get('q', '*')
sort = request.args.get('sort')
# we want the results per page to be constant between pages
# TODO - move this functionality into build_query in util
query = build_query(q, size=RESULTS_PER_PAGE, start=start, sort=sort)
else:
query = json.loads(unquote(json_query))
query['from'] = start
query['size'] = RESULTS_PER_PAGE
# Aggregations are expensive, and we really don't want to
# execute them if they won't be used
for field in ['aggs', 'aggregations']:
if query.get(field):
del query[field]
q = query # Do we really want to display this?
try:
search_results = search.search_share(query)
except MalformedQueryError:
raise HTTPError(http.BAD_REQUEST)
except IndexNotFoundError:
search_results = {'count': 0, 'results': []}
atom_url = api_url_for('search_share_atom', _xml=True, _absolute=True)
return util.create_atom_feed(name='SHARE',
data=search_results['results'],
query=q,
size=RESULTS_PER_PAGE,
start=start,
url=atom_url,
to_atom=share_search.to_atom)
def search_share():
tick = time.time()
results = {}
count = request.args.get('count') is not None
raw = request.args.get('raw') is not None
if request.method == 'POST':
query = request.get_json()
elif request.method == 'GET':
query = build_query(request.args.get('q', '*'),
request.args.get('from'),
request.args.get('size'),
sort=request.args.get('sort'))
if count:
results = search.count_share(query)
else:
results = search.search_share(query, raw)
results['time'] = round(time.time() - tick, 2)
return results
def search_share():
tick = time.time()
results = {}
count = request.args.get('count') is not None
raw = request.args.get('raw') is not None
if request.method == 'POST':
query = request.get_json()
elif request.method == 'GET':
query = build_query(
request.args.get('q', '*'),
request.args.get('from'),
request.args.get('size'),
sort=request.args.get('sort')
)
if count:
results = search.count_share(query)
else:
results = search.search_share(query, raw)
results['time'] = round(time.time() - tick, 2)
return results
def search_share_stats():
q = request.args.get('q')
query = build_query(q, 0, 0) if q else {}
return search.share_stats(query=query)
def search_contributor(query, page=0, size=10, exclude=None, current_user=None):
"""Search for contributors to add to a project using elastic search. Request must
include JSON data with a "query" field.
:param query: The substring of the username to search for
:param page: For pagination, the page number to use for results
:param size: For pagination, the number of results per page
:param exclude: A list of User objects to exclude from the search
:param current_user: A User object of the current user
:return: List of dictionaries, each containing the ID, full name,
most recent employment and education, gravatar URL of an OSF user
"""
start = page * size
items = re.split(r"[\s-]+", query)
exclude = exclude or []
normalized_items = []
for item in items:
try:
normalized_item = six.u(item)
except TypeError:
normalized_item = item
normalized_item = unicodedata.normalize("NFKD", normalized_item).encode("ascii", "ignore")
normalized_items.append(normalized_item)
items = normalized_items
query = " AND ".join("{}*~".format(re.escape(item)) for item in items) + "".join(
' NOT id:"{}"'.format(excluded._id) for excluded in exclude
)
results = search(build_query(query, start=start, size=size), index=INDEX, doc_type="user")
docs = results["results"]
pages = math.ceil(results["counts"].get("user", 0) / size)
validate_page_num(page, pages)
users = []
for doc in docs:
# TODO: use utils.serialize_user
user = User.load(doc["id"])
if current_user and current_user._id == user._id:
n_projects_in_common = -1
elif current_user:
n_projects_in_common = current_user.n_projects_in_common(user)
else:
n_projects_in_common = 0
if user is None:
logger.error("Could not load user {0}".format(doc["id"]))
continue
if user.is_active: # exclude merged, unregistered, etc.
current_employment = None
education = None
if user.jobs:
current_employment = user.jobs[0]["institution"]
if user.schools:
education = user.schools[0]["institution"]
users.append(
{
"fullname": doc["user"],
"id": doc["id"],
"employment": current_employment,
"education": education,
"n_projects_in_common": n_projects_in_common,
"gravatar_url": gravatar(user, use_ssl=True, size=settings.PROFILE_IMAGE_MEDIUM),
"profile_url": user.profile_url,
"registered": user.is_registered,
"active": user.is_active,
}
)
return {"users": users, "total": results["counts"]["total"], "pages": pages, "page": page}
def search_contributor(query, page=0, size=10, exclude=[], current_user=None):
"""Search for contributors to add to a project using elastic search. Request must
include JSON data with a "query" field.
:param query: The substring of the username to search for
:param page: For pagination, the page number to use for results
:param size: For pagination, the number of results per page
:param exclude: A list of User objects to exclude from the search
:param current_user: A User object of the current user
:return: List of dictionaries, each containing the ID, full name,
most recent employment and education, gravatar URL of an OSF user
"""
start = (page * size)
items = re.split(r'[\s-]+', query)
query = ''
query = " AND ".join('{}*~'.format(re.escape(item)) for item in items) + \
"".join(' NOT "{}"'.format(excluded) for excluded in exclude)
results = search(build_query(query, start=start, size=size), index=INDEX, doc_type='user')
docs = results['results']
pages = math.ceil(results['counts'].get('user', 0) / size)
users = []
for doc in docs:
# TODO: use utils.serialize_user
user = User.load(doc['id'])
if current_user:
n_projects_in_common = current_user.n_projects_in_common(user)
else:
n_projects_in_common = 0
if user is None:
logger.error('Could not load user {0}'.format(doc['id']))
continue
if user.is_active: # exclude merged, unregistered, etc.
current_employment = None
education = None
if user.jobs:
current_employment = user.jobs[0]['institution']
if user.schools:
education = user.schools[0]['institution']
users.append({
'fullname': doc['user'],
'id': doc['id'],
'employment': current_employment,
'education': education,
'n_projects_in_common': n_projects_in_common,
'gravatar_url': gravatar(
user,
use_ssl=True,
size=settings.GRAVATAR_SIZE_ADD_CONTRIBUTOR,
),
'profile_url': user.profile_url,
'registered': user.is_registered,
'active': user.is_active
})
return {
'users': users,
'total': results['counts']['total'],
'pages': pages,
'page': page,
}
def query(term):
results = search.search(build_query(term))
return results
def search_contributor(query, page=0, size=10, exclude=None, current_user=None):
"""Search for contributors to add to a project using elastic search. Request must
include JSON data with a "query" field.
:param query: The substring of the username to search for
:param page: For pagination, the page number to use for results
:param size: For pagination, the number of results per page
:param exclude: A list of User objects to exclude from the search
:param current_user: A User object of the current user
:return: List of dictionaries, each containing the ID, full name,
most recent employment and education, profile_image URL of an OSF user
"""
start = (page * size)
items = re.split(r'[\s-]+', query)
exclude = exclude or []
normalized_items = []
for item in items:
try:
normalized_item = six.u(item)
except TypeError:
normalized_item = item
normalized_item = unicodedata.normalize('NFKD', normalized_item).encode('ascii', 'ignore')
normalized_items.append(normalized_item)
items = normalized_items
query = ' AND '.join('{}*~'.format(re.escape(item)) for item in items) + \
''.join(' NOT id:"{}"'.format(excluded._id) for excluded in exclude)
results = search(build_query(query, start=start, size=size), index=INDEX, doc_type='user')
docs = results['results']
pages = math.ceil(results['counts'].get('user', 0) / size)
validate_page_num(page, pages)
users = []
for doc in docs:
# TODO: use utils.serialize_user
user = OSFUser.load(doc['id'])
if current_user and current_user._id == user._id:
n_projects_in_common = -1
elif current_user:
n_projects_in_common = current_user.n_projects_in_common(user)
else:
n_projects_in_common = 0
if user is None:
logger.error('Could not load user {0}'.format(doc['id']))
continue
if user.is_active: # exclude merged, unregistered, etc.
current_employment = None
education = None
if user.jobs:
current_employment = user.jobs[0]['institution']
if user.schools:
education = user.schools[0]['institution']
users.append({
'fullname': doc['user'],
'id': doc['id'],
'employment': current_employment,
'education': education,
'social': user.social_links,
'n_projects_in_common': n_projects_in_common,
'profile_image_url': profile_image_url(settings.PROFILE_IMAGE_PROVIDER,
user,
use_ssl=True,
size=settings.PROFILE_IMAGE_MEDIUM),
'profile_url': user.profile_url,
'registered': user.is_registered,
'active': user.is_active
})
return {
'users': users,
'total': results['counts']['total'],
'pages': pages,
'page': page,
}
def query(term):
results = search.search(build_query(term))
return results
def query(term, raw=False):
results = search.search(build_query(term),
index=elastic_search.INDEX,
raw=raw)
return results
def query(term):
results = search.search(build_query(term), index=elastic_search.INDEX)
return results
def search_share_stats():
q = request.args.get('q')
query = build_query(q, 0, 0) if q else {}
return search.share_stats(query=query)
def query(term, raw=False):
results = search.search(build_query(term), index=elastic_search.INDEX, raw=raw)
return results
def search_contributor(query,
page=0,
size=10,
exclude=None,
current_user=None):
"""Search for contributors to add to a project using elastic search. Request must
include JSON data with a "query" field.
:param query: The substring of the username to search for
:param page: For pagination, the page number to use for results
:param size: For pagination, the number of results per page
:param exclude: A list of User objects to exclude from the search
:param current_user: A User object of the current user
:return: List of dictionaries, each containing the ID, full name,
most recent employment and education, profile_image URL of an OSF user
"""
start = (page * size)
items = re.split(r'[\s-]+', query)
exclude = exclude or []
normalized_items = []
for item in items:
try:
normalized_item = six.u(item)
except TypeError:
normalized_item = item
normalized_item = unicodedata.normalize('NFKD',
normalized_item).encode(
'ascii', 'ignore')
normalized_items.append(normalized_item)
items = normalized_items
query = ' AND '.join('{}*~'.format(re.escape(item)) for item in items) + \
''.join(' NOT id:"{}"'.format(excluded._id) for excluded in exclude)
results = search(build_query(query, start=start, size=size),
index=INDEX,
doc_type='user')
docs = results['results']
pages = math.ceil(results['counts'].get('user', 0) / size)
validate_page_num(page, pages)
users = []
for doc in docs:
# TODO: use utils.serialize_user
user = OSFUser.load(doc['id'])
if current_user and current_user._id == user._id:
n_projects_in_common = -1
elif current_user:
n_projects_in_common = current_user.n_projects_in_common(user)
else:
n_projects_in_common = 0
if user is None:
logger.error('Could not load user {0}'.format(doc['id']))
continue
if user.is_active: # exclude merged, unregistered, etc.
current_employment = None
education = None
if user.jobs:
current_employment = user.jobs[0]['institution']
if user.schools:
education = user.schools[0]['institution']
users.append({
'fullname':
doc['user'],
'id':
doc['id'],
'employment':
current_employment,
'education':
education,
'social':
user.social_links,
'n_projects_in_common':
n_projects_in_common,
'profile_image_url':
profile_image_url(settings.PROFILE_IMAGE_PROVIDER,
user,
use_ssl=True,
size=settings.PROFILE_IMAGE_MEDIUM),
'profile_url':
user.profile_url,
'registered':
user.is_registered,
'active':
user.is_active
})
return {
'users': users,
'total': results['counts']['total'],
'pages': pages,
'page': page,
}
def query(term):
results = search.search(build_query(term), index=elastic_search.INDEX)
return results