def search_share_atom(**kwargs): q = request.args.get('q', '*') sort = request.args.get('sort', 'dateUpdated') # we want the results per page to be constant between pages # TODO - move this functionality into build_query in util start = util.compute_start(request.args.get('page', 1), RESULTS_PER_PAGE) query = build_query(q, size=RESULTS_PER_PAGE, start=start, sort=sort) try: search_results = search.search_share(query, index='share_v1') except MalformedQueryError: raise HTTPError(http.BAD_REQUEST) except IndexNotFoundError: search_results = {'count': 0, 'results': []} atom_url = api_url_for('search_share_atom', _xml=True, _absolute=True) return util.create_atom_feed(name='SHARE', data=search_results['results'], query=q, size=RESULTS_PER_PAGE, start=start, url=atom_url, to_atom=share_search.to_atom)
def search_share_atom(**kwargs): q = request.args.get('q', '*') sort = request.args.get('sort', 'dateUpdated') # we want the results per page to be constant between pages # TODO - move this functionality into build_query in util start = util.compute_start(request.args.get('page', 1), RESULTS_PER_PAGE) query = build_query(q, size=RESULTS_PER_PAGE, start=start, sort=sort) try: search_results = search.search_share(query) except MalformedQueryError: raise HTTPError(http.BAD_REQUEST) except IndexNotFoundError: search_results = { 'count': 0, 'results': [] } atom_url = api_url_for('search_share_atom', _xml=True, _absolute=True) return util.create_atom_feed( name='SHARE', data=search_results['results'], query=q, size=RESULTS_PER_PAGE, start=start, url=atom_url, to_atom=share_search.to_atom )
def search_share(): tick = time.time() results = {} count = request.args.get('count') is not None raw = request.args.get('raw') is not None version = request.args.get('v') if version: index = settings.SHARE_ELASTIC_INDEX_TEMPLATE.format(version) else: index = settings.SHARE_ELASTIC_INDEX if request.method == 'POST': query = request.get_json() elif request.method == 'GET': query = build_query(request.args.get('q', '*'), request.args.get('from', 0), request.args.get('size', 10), sort=request.args.get('sort')) if count: results = search.count_share(query, index=index) else: results = search.search_share(query, raw, index=index) results['time'] = round(time.time() - tick, 2) return results
def search_share(): tick = time.time() results = {} count = request.args.get('count') is not None raw = request.args.get('raw') is not None version = request.args.get('v') if version: index = settings.SHARE_ELASTIC_INDEX_TEMPLATE.format(version) else: index = settings.SHARE_ELASTIC_INDEX if request.method == 'POST': query = request.get_json() elif request.method == 'GET': query = build_query( request.args.get('q', '*'), request.args.get('from', 0), request.args.get('size', 10), sort=request.args.get('sort') ) if count: results = search.count_share(query, index=index) else: results = search.search_share(query, raw, index=index) results['time'] = round(time.time() - tick, 2) return results
def get_queryset(self): query = self.request.query_params.get('q', '*') page = int(self.request.query_params.get('page', '1')) page_size = min(int(self.request.query_params.get('page[size]', REST_FRAMEWORK['PAGE_SIZE'])), MAX_PAGE_SIZE) start = (page - 1) * page_size try: results = search.search(build_query(query, start=start, size=page_size), doc_type=self.doc_type, raw=True) except MalformedQueryError as e: raise ValidationError(e.message) return results
def get_queryset(self, query=None): page = int(self.request.query_params.get('page', '1')) page_size = min(int(self.request.query_params.get('page[size]', REST_FRAMEWORK['PAGE_SIZE'])), MAX_PAGE_SIZE) start = (page - 1) * page_size if query: # Parser has built query, but needs paging info query['from'] = start query['size'] = page_size else: query = build_query(self.request.query_params.get('q', '*'), start=start, size=page_size) try: results = search.search(query, doc_type=self.doc_type, raw=True) except MalformedQueryError as e: raise ValidationError(e.message) return results
def search_search(**kwargs): _type = kwargs.get('type', None) tick = time.time() results = {} if request.method == 'POST': results = search.search(request.get_json(), doc_type=_type) elif request.method == 'GET': q = request.args.get('q', '*') # TODO Match javascript params? start = request.args.get('from', '0') size = request.args.get('size', '10') results = search.search(build_query(q, start, size), doc_type=_type) results['time'] = round(time.time() - tick, 2) return results
def search_share_atom(**kwargs): json_query = request.args.get('jsonQuery') start = util.compute_start(request.args.get('page', 1), RESULTS_PER_PAGE) if not json_query: q = request.args.get('q', '*') sort = request.args.get('sort') # we want the results per page to be constant between pages # TODO - move this functionality into build_query in util query = build_query(q, size=RESULTS_PER_PAGE, start=start, sort=sort) else: query = json.loads(unquote(json_query)) query['from'] = start query['size'] = RESULTS_PER_PAGE # Aggregations are expensive, and we really don't want to # execute them if they won't be used for field in ['aggs', 'aggregations']: if query.get(field): del query[field] q = query # Do we really want to display this? try: search_results = search.search_share(query) except MalformedQueryError: raise HTTPError(http.BAD_REQUEST) except IndexNotFoundError: search_results = { 'count': 0, 'results': [] } atom_url = api_url_for('search_share_atom', _xml=True, _absolute=True) return util.create_atom_feed( name='SHARE', data=search_results['results'], query=q, size=RESULTS_PER_PAGE, start=start, url=atom_url, to_atom=share_search.to_atom )
def search_share_atom(**kwargs): json_query = request.args.get('jsonQuery') start = util.compute_start(request.args.get('page', 1), RESULTS_PER_PAGE) if not json_query: q = request.args.get('q', '*') sort = request.args.get('sort') # we want the results per page to be constant between pages # TODO - move this functionality into build_query in util query = build_query(q, size=RESULTS_PER_PAGE, start=start, sort=sort) else: query = json.loads(unquote(json_query)) query['from'] = start query['size'] = RESULTS_PER_PAGE # Aggregations are expensive, and we really don't want to # execute them if they won't be used for field in ['aggs', 'aggregations']: if query.get(field): del query[field] q = query # Do we really want to display this? try: search_results = search.search_share(query) except MalformedQueryError: raise HTTPError(http.BAD_REQUEST) except IndexNotFoundError: search_results = {'count': 0, 'results': []} atom_url = api_url_for('search_share_atom', _xml=True, _absolute=True) return util.create_atom_feed(name='SHARE', data=search_results['results'], query=q, size=RESULTS_PER_PAGE, start=start, url=atom_url, to_atom=share_search.to_atom)
def search_share(): tick = time.time() results = {} count = request.args.get('count') is not None raw = request.args.get('raw') is not None if request.method == 'POST': query = request.get_json() elif request.method == 'GET': query = build_query(request.args.get('q', '*'), request.args.get('from'), request.args.get('size'), sort=request.args.get('sort')) if count: results = search.count_share(query) else: results = search.search_share(query, raw) results['time'] = round(time.time() - tick, 2) return results
def search_share(): tick = time.time() results = {} count = request.args.get('count') is not None raw = request.args.get('raw') is not None if request.method == 'POST': query = request.get_json() elif request.method == 'GET': query = build_query( request.args.get('q', '*'), request.args.get('from'), request.args.get('size'), sort=request.args.get('sort') ) if count: results = search.count_share(query) else: results = search.search_share(query, raw) results['time'] = round(time.time() - tick, 2) return results
def search_share_stats(): q = request.args.get('q') query = build_query(q, 0, 0) if q else {} return search.share_stats(query=query)
def search_contributor(query, page=0, size=10, exclude=None, current_user=None): """Search for contributors to add to a project using elastic search. Request must include JSON data with a "query" field. :param query: The substring of the username to search for :param page: For pagination, the page number to use for results :param size: For pagination, the number of results per page :param exclude: A list of User objects to exclude from the search :param current_user: A User object of the current user :return: List of dictionaries, each containing the ID, full name, most recent employment and education, gravatar URL of an OSF user """ start = page * size items = re.split(r"[\s-]+", query) exclude = exclude or [] normalized_items = [] for item in items: try: normalized_item = six.u(item) except TypeError: normalized_item = item normalized_item = unicodedata.normalize("NFKD", normalized_item).encode("ascii", "ignore") normalized_items.append(normalized_item) items = normalized_items query = " AND ".join("{}*~".format(re.escape(item)) for item in items) + "".join( ' NOT id:"{}"'.format(excluded._id) for excluded in exclude ) results = search(build_query(query, start=start, size=size), index=INDEX, doc_type="user") docs = results["results"] pages = math.ceil(results["counts"].get("user", 0) / size) validate_page_num(page, pages) users = [] for doc in docs: # TODO: use utils.serialize_user user = User.load(doc["id"]) if current_user and current_user._id == user._id: n_projects_in_common = -1 elif current_user: n_projects_in_common = current_user.n_projects_in_common(user) else: n_projects_in_common = 0 if user is None: logger.error("Could not load user {0}".format(doc["id"])) continue if user.is_active: # exclude merged, unregistered, etc. current_employment = None education = None if user.jobs: current_employment = user.jobs[0]["institution"] if user.schools: education = user.schools[0]["institution"] users.append( { "fullname": doc["user"], "id": doc["id"], "employment": current_employment, "education": education, "n_projects_in_common": n_projects_in_common, "gravatar_url": gravatar(user, use_ssl=True, size=settings.PROFILE_IMAGE_MEDIUM), "profile_url": user.profile_url, "registered": user.is_registered, "active": user.is_active, } ) return {"users": users, "total": results["counts"]["total"], "pages": pages, "page": page}
def search_contributor(query, page=0, size=10, exclude=[], current_user=None): """Search for contributors to add to a project using elastic search. Request must include JSON data with a "query" field. :param query: The substring of the username to search for :param page: For pagination, the page number to use for results :param size: For pagination, the number of results per page :param exclude: A list of User objects to exclude from the search :param current_user: A User object of the current user :return: List of dictionaries, each containing the ID, full name, most recent employment and education, gravatar URL of an OSF user """ start = (page * size) items = re.split(r'[\s-]+', query) query = '' query = " AND ".join('{}*~'.format(re.escape(item)) for item in items) + \ "".join(' NOT "{}"'.format(excluded) for excluded in exclude) results = search(build_query(query, start=start, size=size), index=INDEX, doc_type='user') docs = results['results'] pages = math.ceil(results['counts'].get('user', 0) / size) users = [] for doc in docs: # TODO: use utils.serialize_user user = User.load(doc['id']) if current_user: n_projects_in_common = current_user.n_projects_in_common(user) else: n_projects_in_common = 0 if user is None: logger.error('Could not load user {0}'.format(doc['id'])) continue if user.is_active: # exclude merged, unregistered, etc. current_employment = None education = None if user.jobs: current_employment = user.jobs[0]['institution'] if user.schools: education = user.schools[0]['institution'] users.append({ 'fullname': doc['user'], 'id': doc['id'], 'employment': current_employment, 'education': education, 'n_projects_in_common': n_projects_in_common, 'gravatar_url': gravatar( user, use_ssl=True, size=settings.GRAVATAR_SIZE_ADD_CONTRIBUTOR, ), 'profile_url': user.profile_url, 'registered': user.is_registered, 'active': user.is_active }) return { 'users': users, 'total': results['counts']['total'], 'pages': pages, 'page': page, }
def query(term): results = search.search(build_query(term)) return results
def search_contributor(query, page=0, size=10, exclude=None, current_user=None): """Search for contributors to add to a project using elastic search. Request must include JSON data with a "query" field. :param query: The substring of the username to search for :param page: For pagination, the page number to use for results :param size: For pagination, the number of results per page :param exclude: A list of User objects to exclude from the search :param current_user: A User object of the current user :return: List of dictionaries, each containing the ID, full name, most recent employment and education, profile_image URL of an OSF user """ start = (page * size) items = re.split(r'[\s-]+', query) exclude = exclude or [] normalized_items = [] for item in items: try: normalized_item = six.u(item) except TypeError: normalized_item = item normalized_item = unicodedata.normalize('NFKD', normalized_item).encode('ascii', 'ignore') normalized_items.append(normalized_item) items = normalized_items query = ' AND '.join('{}*~'.format(re.escape(item)) for item in items) + \ ''.join(' NOT id:"{}"'.format(excluded._id) for excluded in exclude) results = search(build_query(query, start=start, size=size), index=INDEX, doc_type='user') docs = results['results'] pages = math.ceil(results['counts'].get('user', 0) / size) validate_page_num(page, pages) users = [] for doc in docs: # TODO: use utils.serialize_user user = OSFUser.load(doc['id']) if current_user and current_user._id == user._id: n_projects_in_common = -1 elif current_user: n_projects_in_common = current_user.n_projects_in_common(user) else: n_projects_in_common = 0 if user is None: logger.error('Could not load user {0}'.format(doc['id'])) continue if user.is_active: # exclude merged, unregistered, etc. current_employment = None education = None if user.jobs: current_employment = user.jobs[0]['institution'] if user.schools: education = user.schools[0]['institution'] users.append({ 'fullname': doc['user'], 'id': doc['id'], 'employment': current_employment, 'education': education, 'social': user.social_links, 'n_projects_in_common': n_projects_in_common, 'profile_image_url': profile_image_url(settings.PROFILE_IMAGE_PROVIDER, user, use_ssl=True, size=settings.PROFILE_IMAGE_MEDIUM), 'profile_url': user.profile_url, 'registered': user.is_registered, 'active': user.is_active }) return { 'users': users, 'total': results['counts']['total'], 'pages': pages, 'page': page, }
def query(term, raw=False): results = search.search(build_query(term), index=elastic_search.INDEX, raw=raw) return results
def query(term): results = search.search(build_query(term), index=elastic_search.INDEX) return results
def search_contributor(query, page=0, size=10, exclude=None, current_user=None): """Search for contributors to add to a project using elastic search. Request must include JSON data with a "query" field. :param query: The substring of the username to search for :param page: For pagination, the page number to use for results :param size: For pagination, the number of results per page :param exclude: A list of User objects to exclude from the search :param current_user: A User object of the current user :return: List of dictionaries, each containing the ID, full name, most recent employment and education, profile_image URL of an OSF user """ start = (page * size) items = re.split(r'[\s-]+', query) exclude = exclude or [] normalized_items = [] for item in items: try: normalized_item = six.u(item) except TypeError: normalized_item = item normalized_item = unicodedata.normalize('NFKD', normalized_item).encode( 'ascii', 'ignore') normalized_items.append(normalized_item) items = normalized_items query = ' AND '.join('{}*~'.format(re.escape(item)) for item in items) + \ ''.join(' NOT id:"{}"'.format(excluded._id) for excluded in exclude) results = search(build_query(query, start=start, size=size), index=INDEX, doc_type='user') docs = results['results'] pages = math.ceil(results['counts'].get('user', 0) / size) validate_page_num(page, pages) users = [] for doc in docs: # TODO: use utils.serialize_user user = OSFUser.load(doc['id']) if current_user and current_user._id == user._id: n_projects_in_common = -1 elif current_user: n_projects_in_common = current_user.n_projects_in_common(user) else: n_projects_in_common = 0 if user is None: logger.error('Could not load user {0}'.format(doc['id'])) continue if user.is_active: # exclude merged, unregistered, etc. current_employment = None education = None if user.jobs: current_employment = user.jobs[0]['institution'] if user.schools: education = user.schools[0]['institution'] users.append({ 'fullname': doc['user'], 'id': doc['id'], 'employment': current_employment, 'education': education, 'social': user.social_links, 'n_projects_in_common': n_projects_in_common, 'profile_image_url': profile_image_url(settings.PROFILE_IMAGE_PROVIDER, user, use_ssl=True, size=settings.PROFILE_IMAGE_MEDIUM), 'profile_url': user.profile_url, 'registered': user.is_registered, 'active': user.is_active }) return { 'users': users, 'total': results['counts']['total'], 'pages': pages, 'page': page, }