def list(request):
pur_reqs = Request.objects.all()
in_format = "%m/%d/%Y"
db_format = "%Y-%m-%d"
# Date filters
if "start" in request.GET and validate_date_input(request.GET["start"]):
st = datetime.strptime(request.GET["start"], in_format)
st = st.astimezone(pytz.timezone(settings.TIME_ZONE)).strftime(db_format)
pur_reqs = pur_reqs.filter(timestamp__gte=st)
if "end" in request.GET and validate_date_input(request.GET["end"]):
et = datetime.strptime(request.GET["end"], in_format)
et += timedelta(days=1)
et = et.astimezone(pytz.timezone(settings.TIME_ZONE)).strftime(db_format)
pur_reqs = pur_reqs.filter(timestamp__lt=et)
# Approval filters
final_query = Q()
if "den" in request.GET:
final_query |= Q(approved=False)
if "und" in request.GET:
final_query |= Q(approved=None)
if "app" in request.GET:
final_query |= Q(approved=True, ordered=False, delivered=False)
if "ord" in request.GET:
final_query |= Q(ordered=True, delivered=False)
if "del" in request.GET:
final_query |= Q(delivered=True)
pur_reqs = pur_reqs.filter(final_query)
# Search bar
if "q" in request.GET and request.GET["q"]:
search_terms = request.GET["q"].split()
product_terms = []
user_terms = []
supplier_terms = []
for term in search_terms:
if term.startswith("user:"******"supp:"):
supplier_terms.append(term[5:])
elif term.startswith("supplier:"):
supplier_terms.append(term[9:])
else:
product_terms.append(term)
for p_term in product_terms:
pur_reqs = pur_reqs.annotate(
search=SearchVector("item")
).filter(search=p_term)
user_or_query = Q()
for u_term in user_terms:
user_or_query |= Q(author__username=u_term)
pur_reqs = pur_reqs.filter(user_or_query)
supp_or_query = Q()
for supp_term in supplier_terms:
supp_or_query |= Q(supplier__icontains=supp_term)
pur_reqs = pur_reqs.filter(supp_or_query)
submitted_filters = {
"start": request.GET["start"] if "start" in request.GET and validate_date_input(request.GET["start"]) else "",
"end": request.GET["end"] if "end" in request.GET and validate_date_input(request.GET["end"]) else "",
"q": request.GET["q"] if "q" in request.GET else "",
"app_checked": "app" in request.GET,
"und_checked": "und" in request.GET,
"den_checked": "den" in request.GET,
"ord_checked": "ord" in request.GET,
"del_checked": "del" in request.GET,
}
context = {
'pur_req_list': pur_reqs.order_by('-timestamp'),
'filters': submitted_filters,
'theme_color': settings.THEME_COLOR,
'team_name': settings.TEAM_NAME,
}
return render(request, "purchaseRequests/list.html", context)
def test_simple_on_scene(self):
searched = Line.objects.annotate(
search=SearchVector('scene__setting', 'dialogue'),
).filter(search='Forest')
self.assertSequenceEqual(searched, self.verses)
def test_search_two_terms(self):
searched = Line.objects.annotate(
search=SearchVector('scene__setting', 'dialogue'),
).filter(search='heart forest')
self.assertSequenceEqual(searched, [self.verse2])
from fedsearch.pkgsearch.models import Package
from django.contrib.postgres.search import SearchVector
search_vector = (SearchVector("name", weight="A", config="english") +
SearchVector("summary", weight="B", config="english") +
SearchVector("description", weight="D", config="english"))
def create_pkgs():
""" Helper method used to create instances of Package and SubPackage objects """
package1, created = Package.objects.get_or_create(
name="mate-terminal",
summary="Terminal emulator for MATE",
description="Mate-terminal is a terminal emulator for MATE. \
It supports translucent backgrounds, opening multiple terminals \
in a single window (tabs) and clickable URLs.",
point_of_contact="asterix",
icon="",
upstream_url="http://mate-desktop.org/",
search_vector=None,
)
package2, created = Package.objects.get_or_create(
name="gnome-terminal",
summary="Terminal emulator for GNOME",
description="gnome-terminal is a terminal emulator for GNOME. \
It features the ability to use multiple terminals in a single window (tabs) \
and profiles support.",
point_of_contact="obelix",
icon="",
upstream_url="http://www.gnome.org/",
search_vector=None,
def test_existing_vector(self):
Line.objects.update(dialogue_search_vector=SearchVector('dialogue'))
searched = Line.objects.filter(dialogue_search_vector=SearchQuery('Robin killed'))
self.assertSequenceEqual(searched, [self.verse0])
ADD,
MUL,
OR,
get_content_type_pk,
get_descendants_content_types_pks,
)
from ...base import (
BaseSearchBackend,
BaseSearchQueryCompiler,
BaseSearchResults,
FilterFieldError,
)
from .query import Lexeme
from .weights import get_sql_weights, get_weight
EMPTY_VECTOR = SearchVector(Value("", output_field=TextField()))
class ObjectIndexer:
"""
Responsible for extracting data from an object to be inserted into the index.
"""
def __init__(self, obj, backend):
self.obj = obj
self.search_fields = obj.get_search_fields()
self.config = backend.config
self.autocomplete_config = backend.autocomplete_config
def prepare_value(self, value):
if isinstance(value, str):
def populate_db():
for abbr, state in (("ak", "Alaska"), ("wy", "Wyoming")):
d = Division.objects.create(id="ocd-division/country:us/state:" + abbr,
name=state)
j = Jurisdiction.objects.create(
id="ocd-jurisdiction/country:us/state:" + abbr + "/government",
name=state,
division=d,
)
j.legislative_sessions.create(identifier="2017",
name="2017",
start_date="2017-01-01")
j.legislative_sessions.create(identifier="2018",
name="2018",
start_date="2018-01-01")
leg = Organization.objects.create(jurisdiction=j,
classification="legislature",
name=state + " Legislature")
Organization.objects.create(jurisdiction=j,
parent=leg,
classification="lower",
name=state + " House")
Organization.objects.create(jurisdiction=j,
parent=leg,
classification="upper",
name=state + " Senate")
alaska = Jurisdiction.objects.get(name="Alaska")
house = alaska.organizations.get(classification="lower")
# AK House
amanda = make_person("Amanda Adams", "Alaska", "lower", "1", "Republican")
birch = make_person("Bob Birch", "Alaska", "lower", "2", "Republican")
carrie = make_person("Carrie Carr", "Alaska", "lower", "3", "Democratic")
don = make_person("Don Dingle", "Alaska", "lower", "4", "Republican")
# AK Senate
ellen = make_person("Ellen Evil", "Alaska", "upper", "A", "Independent")
make_person("Frank Fur", "Alaska", "upper", "B", "Democratic")
# Ellen used to be a house member
post = house.posts.create(label="5", role="Representative")
ellen.memberships.create(post=post,
organization=house,
end_date="2017-01-01")
# retired house member
rhonda = make_person("Rhonda Retired", "Alaska", "upper", "B",
"Democratic")
for m in rhonda.memberships.all():
m.end_date = "2017-01-01"
m.save()
rhonda.current_role = None
rhonda.save()
# WY House (multi-member districts)
make_person("Greta Gonzalez", "Wyoming", "lower", "1", "Democratic")
make_person("Hank Horn", "Wyoming", "lower", "1", "Republican")
session = alaska.legislative_sessions.get(identifier="2018")
b1 = Bill.objects.create(
id="ocd-bill/1",
title="Moose Freedom Act",
identifier="HB 1",
legislative_session=session,
from_organization=house,
classification=["bill", "constitutional amendment"],
subject=["nature"],
)
b1.abstracts.create(
abstract="Grants all moose equal rights under the law.")
b1.abstracts.create(abstract="Ensure moose freedom.")
b1.other_titles.create(title="Moosemendment")
b1.other_titles.create(title="Moose & Reindeer Freedom Act")
b1.other_titles.create(title="M.O.O.S.E.")
b1.other_identifiers.create(identifier="HCA 1")
b1.other_identifiers.create(identifier="SB 1")
a = b1.actions.create(description="Introduced",
order=10,
organization=house,
date="2018-01-01")
a.related_entities.create(name="Amanda Adams",
entity_type="person",
person=amanda)
b1.actions.create(description="Amended",
order=20,
organization=house,
date="2018-02-01")
a = b1.actions.create(
description="Passed House",
order=30,
organization=house,
date="2018-03-01",
classification=["passage"],
)
a.related_entities.create(name="House",
entity_type="organization",
organization=house)
b1.sponsorships.create(primary=True,
classification="sponsor",
name="Adam One")
b1.sponsorships.create(primary=False,
classification="cosponsor",
name="Beth Two",
person=amanda)
d = b1.documents.create(note="Fiscal Note")
d.links.create(url="https://example.com/fn")
d = b1.documents.create(note="Legal Justification")
d.links.create(url="https://example.com/lj")
d = b1.versions.create(note="First Draft", date="2017-01-01")
dlink = d.links.create(url="https://example.com/1.txt",
media_type="text/plain")
d.links.create(url="https://example.com/1.pdf",
media_type="application/pdf")
d = b1.versions.create(note="Final Draft", date="2017-06-01")
d.links.create(url="https://example.com/f.txt", media_type="text/plain")
d.links.create(url="https://example.com/f.pdf",
media_type="application/pdf")
b1.sources.create(url="https://example.com/s1")
b1.sources.create(url="https://example.com/s2")
b1.sources.create(url="https://example.com/s3")
SearchableBill.objects.create(
bill=b1,
version_link=dlink,
all_titles=b1.title,
raw_text=
"shove some text in here and include a weird word like gorgonzola",
is_error=False,
search_vector="",
)
SearchableBill.objects.update(search_vector=(
SearchVector("all_titles", weight="A", config="english") +
SearchVector("raw_text", weight="B", config="english")))
ve = VoteEvent.objects.create(
bill=b1,
legislative_session=session,
motion_text="Vote on House Passage",
organization=house,
result="failed",
)
ve.counts.create(option="yes", value=1)
ve.counts.create(option="no", value=4)
ve.votes.create(option="yes", voter_name="Amanda Adams", voter=amanda)
ve.votes.create(option="no", voter_name="Birch", voter=birch)
ve.votes.create(option="no", voter_name="Carr", voter=carrie)
ve.votes.create(option="no", voter_name="Dingle", voter=don)
ve.votes.create(option="no", voter_name="Speaker")
rb = Bill.objects.create(
id="ocd-bill/3",
title="Alces alces Freedom Act",
identifier="SB 9",
legislative_session=session,
from_organization=alaska.organizations.get(classification="upper"),
classification=["bill", "constitutional amendment"],
subject=["nature"],
)
b1.related_bills.create(
related_bill=rb,
identifier="SB 9",
legislative_session="2018",
relation_type="companion",
)
rb.sponsorships.create(primary=False,
classification="cosponsor",
name="Beth Two",
person=amanda)
for x in range(10):
make_random_bill("Alaska")
for x in range(14):
make_random_bill("Wyoming")
# populate bill computed data
for bill in Bill.objects.all().filter(
legislative_session__jurisdiction__name__in=["Alaska", "Wyoming"]):
update_bill_fields(bill)
def test_single_coalesce_expression(self):
searched = Line.objects.annotate(
search=SearchVector('dialogue')).filter(search='cadeaux')
self.assertNotIn('COALESCE(COALESCE', str(searched.query))
def test_search_with_non_text(self):
searched = Line.objects.annotate(
search=SearchVector('id'), ).filter(search=str(self.crowd.id))
self.assertSequenceEqual(searched, [self.crowd])
def search_questions_for_user(user, query):
questions = Question.objects.all()
return questions.annotate(
search=SearchVector('user', 'title', 'body', 'tags')).filter(
search=query).distinct('pk')
def res_list(request):
search_term = request.GET.get('q')
res_list = []
nounchunks = []
if search_term:
if len(search_term.split(' ')) > 1:
tsquery = ' & '.join(search_term.split(' '))
tsquery_word = ' | '.join(search_term.split(' '))
else:
tsquery = search_term
tsquery_word = search_term
query = ("select id, chunk_en from noun_chunks where "
"to_tsvector(chunk_en) @@ to_tsquery('%s');")
nounchunks += conn.execute(query % tsquery).fetchall()
results = conn.execute(
"select distinct keywords.id, first_name, last_name, word_en, count, affils.name"
" from keywords inner join affils on keywords.affil_scopus_id=affils.scopus_id "
"where to_tsvector(word_en) @@ to_tsquery('%s')"
" order by count desc" % tsquery_word).fetchall()
if results:
for rec in results:
query = (
'select count(*) from abstracts inner join abstract_has_keywords '
'on abstract_has_keywords.abstract_id=abstracts.id '
'inner join keywords on keywords.id=abstract_has_keywords.keyword_id '
'where keywords.id=%d;')
total_abstract = conn.execute(query % int(rec[0])).scalar()
fname = rec[1].replace("'", "\'") if rec[1] else ''
lname = rec[2].replace("'", "\'") if rec[2] else ''
query = (
"select id,scholarship_info_id from authors where lower(first_name)=lower(%s) "
"and lower(last_name)=lower(%s)")
_author = conn.execute(query, (fname, lname)).fetchone()
if _author:
_author_id, _sc_id = _author[0], _author[1]
if _author_id:
if _sc_id:
sc = True
else:
sc = False
res_list.append(
Researcher(_author_id, rec[1], rec[2], rec[3],
rec[4], rec[5], total_abstract, sc))
profiles = {}
for word in search_term.split(' '):
for p in Profile.objects.annotate(
search=SearchVector('field_of_interest')).filter(search=word):
field_of_interest = (f.strip()
for f in p.field_of_interest.split(','))
profiles[p.user.username] = (p.user.first_name, p.user.last_name,
field_of_interest)
authors = []
query = ("select id, first_name,last_name from authors where "
"lower(first_name)=lower(%s) or lower(last_name)=lower(%s);")
for id, first_name, last_name in conn.execute(query,
(search_term, search_term)):
authors.append((id, first_name, last_name))
return render(request,
template_name='analytics/res_list.html',
context={
'search_term': search_term,
'results': res_list,
'nounchunks': nounchunks,
'profiles': profiles,
'authors': authors
})
def fts_search(qstring, qnum):
reviews = FTSReview.objects.annotate(search=SearchVector(
'review_text', 'review_summary'), ).filter(search=qstring)
num = reviews.count()
return SearchResult(reviews[:qnum], {}, num)
def get_queryset(self):
query = self.request.GET.get("q")
return Post.published.annotate(search=SearchVector(
"title", "body"), ).filter(search=query)
def get_related_posts(self):
self.query = [q for q in self.title.split() if len(q) >= 4]
return self.annotate(search=SearchVector(
'title', 'text_description'), ).filter(search=self.query[0])
def append(self, text, weight=None):
self.text.append(text)
new_vector = SearchVector(Value(text), weight=weight)
self.vector = self.vector + new_vector
def test_config_query_explicit(self):
searched = Line.objects.annotate(search=SearchVector(
'scene__setting', 'dialogue', config='french'), ).filter(
search=SearchQuery('cadeaux', config='french'))
self.assertSequenceEqual(searched, [self.french])
event.partner = 'CTCN'
event.title = result['Title']
event.description = result['Body']
event.language = 'english'
event.keywords = ' '.join(result['CTCN Keyword Matches'])
event.location = result['Country']
event.raw_data = json.dumps(result)
event.url = result['Path']
if 'value' in result['Date and time'] and 'value2' in result[
'Date and time']:
event.meta_data = {
'from': result['Date and time']['value'],
'to': result['Date and time']['value2']
}
event.save()
event.search_vector = SearchVector(
'title', weight='A', config='english') + SearchVector(
'keywords', weight='D', config='english') + SearchVector(
'description', weight='C', config='english')
event.save()
print(event.client_id, event.title)
url = 'https://www.ctc-n.org/api/news'
r = requests.get(url)
if r.status_code == 200:
results = json.loads(r.text)
for result in results:
client_id = 'CTCN-' + result['nid']
try:
event = Store.objects.get(client_id=client_id)
except Store.DoesNotExist:
event = Store()
def test_config_from_field_implicit(self):
searched = Line.objects.annotate(search=SearchVector(
'scene__setting', 'dialogue',
config=F('dialogue_config')), ).filter(search='cadeaux')
self.assertSequenceEqual(searched, [self.french])
def update_search_vector(sender, instance, *args, **kwargs):
sender.objects.filter(pk=instance.id).update(search_vector=(
SearchVector('name', weight='A') +
SearchVector('description', weight='B')
))
def test_query_and(self):
searched = Line.objects.annotate(search=SearchVector(
'scene__setting',
'dialogue'), ).filter(search=SearchQuery('bedemir')
& SearchQuery('scales'))
self.assertSequenceEqual(searched, [self.bedemir0])
def update_search_vector(self):
self.search_vector = SearchVector("search_document",
config=settings.MISAGO_SEARCH_CONFIG)
def test_ranking_chaining(self):
searched = Line.objects.filter(character=self.minstrel).annotate(
rank=SearchRank(
SearchVector('dialogue'),
SearchQuery('brave sir robin')), ).filter(rank__gt=0.3)
self.assertSequenceEqual(searched, [self.verse0])
def search(request):
context = {
"title": "Search",
}
if request.method == "GET":
return render(request, "web/search.html", context)
if request.method == "POST":
search_term = request.POST.get("search_term", None)
if not search_term:
return render(request, "web/search.html", context)
results = {}
query = SearchQuery(search_term)
# Pages
vector = SearchVector("title", weight="A") + SearchVector(
"main_content", "sidebar", weight="B")
pages = (Page.objects.annotate(
search=vector,
rank=SearchRank(vector,
query)).filter(search=query).order_by("-rank"))
if pages.exists():
results["pages"] = pages
# Events
vector = (SearchVector("title", weight="A") +
SearchVector("description", weight="B") +
SearchVector("location", weight="C"))
events = (Event.objects.annotate(
search=vector,
rank=SearchRank(vector,
query)).filter(search=query).order_by("rank"))
if events.exists():
results["events"] = events
# Technical Resources
vector = (SearchVector("name", weight="A") +
SearchVector("summary", weight="B") +
SearchVector("source", weight="C"))
tech_resources = (TechnicalResource.objects.annotate(
search=vector,
rank=SearchRank(vector,
query)).filter(search=query).order_by("rank"))
if tech_resources.exists():
results["tech_resources"] = tech_resources
# Funding Resources
vector = (SearchVector("name", weight="A") +
SearchVector("description", weight="B") +
SearchVector("source_name", weight="C"))
funding_resources = (FundingResource.objects.annotate(
search=vector,
rank=SearchRank(vector, query)).filter(search=query).exclude(
due_date__lte=datetime.datetime.today()).order_by("rank"))
if funding_resources.exists():
results["funding_resources"] = funding_resources
context.update([("search_attempt", True), ("search_term", search_term),
("results", results)])
return render(request, "web/search.html", context)
def _save_search_product(self):
return Product.objects.annotate(
search=SearchVector('name'))\
.filter(search=SearchQuery(self.query))
def test_existing_vector_config_explicit(self):
Line.objects.update(dialogue_search_vector=SearchVector('dialogue'))
searched = Line.objects.filter(dialogue_search_vector=SearchQuery('cadeaux', config='french'))
self.assertSequenceEqual(searched, [self.french])
def update_search_vector(sender, instance, **kwargs):
NewsItem.objects.filter(pk=instance.pk).update(
search_vector=SearchVector('title', 'text'))
def test_non_exact_match(self):
searched = Line.objects.annotate(
search=SearchVector('scene__setting', 'dialogue'),
).filter(search='heart')
self.assertSequenceEqual(searched, [self.verse2])
def __init__(self):
self.text = []
self.vector = SearchVector(Value(''))
def test_search_with_null(self):
searched = Line.objects.annotate(
search=SearchVector('scene__setting', 'dialogue'),
).filter(search='bedemir')
self.assertEqual(set(searched), {self.bedemir0, self.bedemir1, self.crowd, self.witch, self.duck})
def get_queryset(self, *args, **kwargs):
q = self.request.GET.get('q')
return Course.objects.annotate(
search=SearchVector('title', weight='A') +
SearchVector('overview', weight='B')).filter(
search=SearchQuery(q, search_type='phrase'))
