def search_now():
""" Check all active searches
"""
today = datetime.now()
for sub in Subscription.objects.all():
found = []
if sub.active and today <= sub.expires:
if not isinstance(sub.user, User):
sub.delete()
continue
searches = Search.objects.filter(user=sub.user)
print "Searching {} for {} at {}".format(
len(searches), sub.user.username,
today.strftime('%Y-%m-%d %H:%M'))
for search in searches:
if search.user:
finds = search_for(search)
for find in finds:
print "Found: {}".format(find)
found.extend(finds)
if found:
send_email(sub.user, found)
def search_items(show_secrets):
request_args = {
key: value.strip()
for key, value in flask.request.values.items()
if value and (key in config.www.search_params)
}
request_keys = set(request_args.keys())
order_by = flask.request.values.get("orderBy", const.DEFAULT_ORDER_BY)
if order_by not in config.www.order_by_keys:
flask.abort(
http.client.BAD_REQUEST,
"Key {order_by} is not supported for ordering".format(
order_by=order_by))
#if request_args is empty, we should render empty search form
if len(request_args) == 0:
flask.abort(http.client.BAD_REQUEST, "No search parameters specified")
found_items = set(items)
for index_to_use in (config.www.indexed_search_params & request_keys):
value_to_use = request_args[index_to_use]
if ((index_to_use in config.parser.list_params)
or (index_to_use in config.parser.keyword_list_params)):
values_to_use = utils.strip_split_list(value_to_use, ",")
else:
values_to_use = [value_to_use]
for value in values_to_use:
if index_to_use == "availability":
value = bib_parser.Availability(value)
indexed_items = set(item_index[index_to_use].get(value, set()))
found_items &= indexed_items
searches = []
try:
for search_key in (config.www.nonindexed_search_params & request_keys):
# argument can be missing or be empty
# both cases should be ignored during search
search_param = request_args[search_key]
if len(search_param) > 0:
param_filter = search.search_for(search_key, search_param)
if param_filter is not None:
searches.append(param_filter)
except Exception as ex:
flask.abort(http.client.BAD_REQUEST,
"Some of the search parameters are wrong: {0}".format(ex))
found_items = list(
sorted(filter(search.and_(searches), found_items),
key=bib_parser.BibItem.key_to_key_func(order_by)))
return flask.render_template("search.html",
found_items=found_items,
show_secrets=show_secrets)
def landing():
""" Landing page """
form = SearchForm(request.form)
if request.method == 'POST' and form.validate():
search = parse_query( form.query.data, form.latitude.data, form.longitude.data, current_user )
if not search.make:
search.delete()
form.query.errors = ["Unknown make or model"]
context = {'user':current_user, 'form':form, 'makes': (', '.join('"' + m.lower() + '"' for m in MAKES))}
return render_template( 'landing.html', **context )
search_for( search )
return redirect( url_for('listings.landing', search = search.pk, page = 0) )
context = { 'user':current_user, 'form':form, 'makes': (', '.join('"' + m.lower() + '"' for m in MAKES))}
return render_template( 'landing.html', **context )
def root(show_secrets):
args_filter = lambda pair: len(pair[1]) > 0
request_args = dict(filter(args_filter, flask.request.args.items()))
request_keys = set(request_args.keys())
#if request_args is empty, we should render empty search form
if len(request_args) == 0:
return flask.render_template(
"index.html",
items=items,
show_secrets=show_secrets
)
found_items = None
for index_to_use in (config.www.indexed_search_params & request_keys):
value_to_use = request_args[index_to_use]
if index_to_use in config.parser.list_params:
values_to_use = utils.strip_split_list(value_to_use, ",")
else:
values_to_use = [value_to_use]
for value in values_to_use:
indexed_items = set(item_index[index_to_use].get(value, set()))
if found_items is None:
found_items = indexed_items
else:
found_items &= indexed_items
searches = []
if found_items is None:
#no index was applied
found_items = items
try:
for search_key in (config.www.nonindexed_search_params & request_keys):
# argument can be missing or be empty
# both cases should be ignored during search
search_param = request_args[search_key]
if len(search_param) > 0:
param_filter = search.search_for(search_key, search_param)
if param_filter is not None:
searches.append(param_filter)
except Exception as ex:
flask.abort(400, "Some of the search parameters are wrong: {0}".format(ex))
if len(searches) > 0:
found_items = list(filter(search.and_(searches), found_items))
return flask.render_template(
"index.html",
found_items=found_items,
search_params=request_args,
show_secrets=show_secrets
)
def search_items_test():
"""
Tests if parsed items can be searched by a bunch of parameters
"""
items = bib_parser.BibParser().parse_string(TEST_ITEMS)
item_index = index.Index(items)
for item in items:
item.process_crossrefs(item_index)
item_index.update(items)
author_search = search.search_for_iterable("author", "Петров")
filtered_items = filter(author_search, items)
eq_(len(list(filtered_items)), 1)
#testing exact match
year_search = search.and_([
search.search_for("year_from", 1825),
search.search_for("year_to", 1825)
])
filtered_items = filter(year_search, items)
eq_(len(list(filtered_items)), 1)
#testing partial intersection
year_search = search.and_([
search.search_for("year_from", 1500),
search.search_for("year_to", 1600)
])
filtered_items = filter(year_search, items)
eq_(len(list(filtered_items)), 1)
#testing inner containment
year_search = search.and_([
search.search_for("year_from", 1499),
search.search_for("year_to", 1501)
])
filtered_items = filter(year_search, items)
eq_(len(list(filtered_items)), 1)
#testing outer containment
year_search = search.and_([
search.search_for("year_from", 1400),
search.search_for("year_to", 1600)
])
filtered_items = filter(year_search, items)
eq_(len(list(filtered_items)), 1)
filtered_items = item_index["keywords"]["grumbling"]
eq_(len(list(filtered_items)), 1)
filtered_items = \
item_index["keywords"]["cinquecento"] & \
item_index["keywords"]["historical dance"]
eq_(len(list(filtered_items)), 1)
def test_search_items(self):
"""
Tests if parsed items can be searched by a bunch of parameters
"""
items = bib_parser.BibParser().parse_string(TEST_ITEMS)
item_index = index.Index(items)
for item in items:
item.process_crossrefs(item_index)
item_index.update(items)
author_search = search.search_for_iterable("author", "Петров")
filtered_items = filter(author_search, items)
self.assertEqual(len(list(filtered_items)), 1)
#testing exact match
year_search = search.and_([
search.search_for("year_from", 1825),
search.search_for("year_to", 1825)
])
filtered_items = filter(year_search, items)
self.assertEqual(len(list(filtered_items)), 1)
#testing partial intersection
year_search = search.and_([
search.search_for("year_from", 1500),
search.search_for("year_to", 1600)
])
filtered_items = filter(year_search, items)
self.assertEqual(len(list(filtered_items)), 1)
#testing inner containment
year_search = search.and_([
search.search_for("year_from", 1499),
search.search_for("year_to", 1501)
])
filtered_items = filter(year_search, items)
self.assertEqual(len(list(filtered_items)), 1)
#testing outer containment
year_search = search.and_([
search.search_for("year_from", 1400),
search.search_for("year_to", 1600)
])
filtered_items = filter(year_search, items)
self.assertEqual(len(list(filtered_items)), 1)
filtered_items = item_index["keywords"]["grumbling"]
self.assertEqual(len(list(filtered_items)), 1)
filtered_items = \
item_index["keywords"]["cinquecento"] & \
item_index["keywords"]["historical dance"]
self.assertEqual(len(list(filtered_items)), 1)
def make_searches_from_metadata(metadata):
"""
Creates dict: {
search_alias_key: callable applicable to an item
},
checking if this item match given metadata
"""
result = {}
equality_searches = ["edition", "number", "part", "langid"]
for search_key in equality_searches:
search_value = metadata.get(search_key, None)
if search_value is None:
continue
result[search_key] = search.search_for_eq(
search_key,
search_value
)
synonym_searches = ["author"]
for search_key in synonym_searches:
search_value = metadata.get(search_key, None)
if search_value is None:
continue
synonym_keys = config.www.search_synonyms.get(search_key) + [search_key]
result[search_key] = search.search_for_synonyms(synonym_keys, search_value)
date_searches = ["year_from", "year_to"]
for search_key in date_searches:
search_value = metadata.get(search_key, None)
if search_value is None:
continue
result[search_key] = search.search_for(
search_key,
search_value
)
title = metadata["title"]
title_regexp = re.compile("^" + re.escape(title))
search_for_itemtitle = search.search_for_string_regexp("title", title_regexp)
search_for_booktitle = search.search_for_string_regexp("booktitle", title_regexp)
result["title"] = search.or_([search_for_itemtitle, search_for_booktitle])
volume = metadata.get("volume", None)
if volume is not None:
search_for_volume = search.search_for_optional_eq(
"volume",
volume
)
search_for_volumes = search.search_for_integer_ge(
"volumes",
volume
)
result["volume"] = search.or_([search_for_volume, search_for_volumes])
return result
def make_searches_from_metadata(metadata):
"""
Creates dict: {
search_alias_key: callable applicable to an item
},
checking if this item match given metadata
"""
result = {}
equality_searches = ["edition", "number", "part", "langid"]
for search_key in equality_searches:
search_value = metadata.get(search_key, None)
if search_value is None:
continue
result[search_key] = search.search_for_eq(search_key, search_value)
synonym_searches = ["author"]
for search_key in synonym_searches:
search_value = metadata.get(search_key, None)
if search_value is None:
continue
synonym_keys = config.www.search_synonyms.get(search_key) + [
search_key
]
result[search_key] = search.search_for_synonyms(
synonym_keys, search_value)
date_searches = ["year_from", "year_to"]
for search_key in date_searches:
search_value = metadata.get(search_key, None)
if search_value is None:
continue
result[search_key] = search.search_for(search_key, search_value)
title = metadata["title"]
title_regexp = re.compile("^" + re.escape(title))
search_for_itemtitle = search.search_for_string_regexp(
"title", title_regexp)
search_for_booktitle = search.search_for_string_regexp(
"booktitle", title_regexp)
result["title"] = search.or_([search_for_itemtitle, search_for_booktitle])
volume = metadata.get("volume", None)
if volume is not None:
search_for_volume = search.search_for_optional_eq("volume", volume)
search_for_volumes = search.search_for_integer_ge("volumes", volume)
result["volume"] = search.or_([search_for_volume, search_for_volumes])
return result
def search_items(show_secrets):
request_args = {
key: value.strip()
for key, value
in flask.request.values.items()
if value and (key in config.www.search_params)
}
request_keys = set(request_args.keys())
order_by = flask.request.values.get("orderBy", const.DEFAULT_ORDER_BY)
if order_by not in config.www.order_by_keys:
flask.abort(http.client.BAD_REQUEST, "Key {order_by} is not supported for ordering".format(
order_by=order_by
))
#if request_args is empty, we should render empty search form
if len(request_args) == 0:
flask.abort(http.client.BAD_REQUEST, "No search parameters specified")
found_items = set(items)
for index_to_use in (config.www.indexed_search_params & request_keys):
value_to_use = request_args[index_to_use]
if (
(index_to_use in config.parser.list_params) or
(index_to_use in config.parser.keyword_list_params)
):
values_to_use = utils.strip_split_list(value_to_use, ",")
else:
values_to_use = [value_to_use]
for value in values_to_use:
if index_to_use == "availability":
value = bib_parser.Availability(value)
indexed_items = set(item_index[index_to_use].get(value, set()))
found_items &= indexed_items
searches = []
try:
for search_key in (config.www.nonindexed_search_params & request_keys):
# argument can be missing or be empty
# both cases should be ignored during search
search_param = request_args[search_key]
if len(search_param) > 0:
param_filter = search.search_for(search_key, search_param)
if param_filter is not None:
searches.append(param_filter)
except Exception as ex:
flask.abort(http.client.BAD_REQUEST, "Some of the search parameters are wrong: {0}".format(ex))
found_items = list(sorted(
filter(search.and_(searches), found_items),
key=bib_parser.BibItem.key_to_key_func(order_by)
))
return flask.render_template(
"search.html",
found_items=found_items,
show_secrets=show_secrets
)
def create_search_from_metadata(metadata: {"str": str}) -> callable:
"""
Creates callable applicable to an item,
checing if this item match given metadata
"""
langid = metadata["langid"]
year = metadata["year"]
title = metadata["title"]
author = metadata.get("author", None)
tome = metadata.get("tome", None)
edition = metadata.get("edition", None)
part = metadata.get("part", None)
#keywords = metadata.get("keywords", None)
title_regexp = re.compile("^" + re.escape(title))
search_for_langid = search.search_for_eq("langid", langid)
search_for_year = search.and_([
search.search_for("year_from", year[0]),
search.search_for("year_to", year[1])
])
search_for_itemtitle = search.search_for_string_regexp("title", title_regexp)
search_for_booktitle = search.search_for_string_regexp("booktitle", title_regexp)
search_for_title = search.or_([search_for_itemtitle, search_for_booktitle])
searches = [
search_for_langid,
search_for_year,
search_for_title,
]
if author is not None:
search_for_author = search.search_for_eq(
"author",
author
)
searches.append(search_for_author)
if tome is not None:
search_for_volume = search.search_for_optional_eq(
"volume",
tome
)
search_for_volumes = search.search_for_integer_ge(
"volumes",
tome
)
searches.append(search.or_([search_for_volume, search_for_volumes]))
if edition is not None:
search_for_edition = search.search_for_eq(
"edition",
edition
)
searches.append(search_for_edition)
if part is not None:
search_for_part = search.search_for_eq(
"part",
part
)
searches.append(search_for_part)
#keywords aren't counted
return search.and_(searches)
def make_searches_from_metadata(metadata):
"""
Creates dict: {
search_alias_key: callable applicable to an item
},
checking if this item match given metadata
"""
result = {}
edition = metadata.get("edition")
if edition is not None:
result["edition"] = search.search_for_eq("edition", edition)
part = metadata.get("part")
if part is not None:
result["part"] = search.search_for_eq("part", part)
number = metadata.get("number")
if number is not None:
result["number"] = search.or_([
search.search_for_eq("number", number),
search.search_for_eq("serial_number", number)
])
subset_searches = ["langid"]
for search_key in subset_searches:
search_value = metadata.get(search_key)
if search_value is None:
continue
result[search_key] = search.search_for_any(search_key, search_value)
synonym_searches = ["author"]
for search_key in synonym_searches:
search_value = metadata.get(search_key, None)
if search_value is None:
continue
synonym_keys = config.www.search_synonyms.get(search_key) + [
search_key
]
result[search_key] = search.search_for_synonyms(
synonym_keys, search_value)
date_searches = ["year_from", "year_to"]
for search_key in date_searches:
search_value = metadata.get(search_key, None)
if search_value is None:
continue
result[search_key] = search.search_for(search_key, search_value)
synonym_prefix_searches = ["title"]
for search_key in synonym_prefix_searches:
search_value = metadata.get(search_key)
if search_value is None:
continue
synonym_keys = config.www.search_synonyms.get(search_key) + [
search_key
]
regexp = re.compile("^" + re.escape(search_value))
result[search_key] = search.or_([
search.search_for_string_regexp(synonym, regexp)
for synonym in synonym_keys
])
volume = metadata.get("volume", None)
if volume is not None:
search_for_volume = search.search_for_optional_eq("volume", volume)
search_for_volumes = search.search_for_integer_ge("volumes", volume)
result["volume"] = search.or_([search_for_volume, search_for_volumes])
return result
def save():
""" Save form from listings
"""
form = request.form
search = Search.objects.get(pk=form['search'])
car = None
cars = None
change = False
if not form['name'] == search.name:
search.name = form['name']
change = True
make = form['make']
# Model comes back as make&model
model = form['model'].split('&')[1]
if model == 'Any':
model = None
trim = form['trim']
if not make == search.make:
search.make = make
change = True
if not model == search.model:
if not model:
search.model = None
else:
car = Car.objects.make_model(make, model)
if car:
search.model = model
change = True
if not trim == search.trim:
search.trim = trim
change = True
if form['year_from'] and not int(form['year_from']) == search.year_from:
search.year_from = int(form['year_from'])
change = True
if form['year_to'] and not int(form['year_to']) == search.year_to:
search.year_to = int(form['year_to'])
change = True
if form['mileage']:
miles = search.get('mileage_max', None)
if miles and search.miles != form['milage']:
search.mileage_min = 0
search.mileage_max = int(form['milage'])
change = True
colors = []
if form['color_1']:
colors.append(form['color_1'])
if form['color_2']:
colors.append(form['color_2'])
if not colors == search.color:
search.color = colors
change = True
if form['option_1'] or form['option_2'] or form['option_3']:
pass
if form['zipcode'] or form['miles']:
if form['miles'] == 'unlimited':
if search.geo:
search.geo = []
change = True
elif not search.zip == form['zipcode'] or not search.distance == int(
form['miles']):
search.set_location(form['zipcode'], int(form['miles']))
search.delete_finds(keep_notes=True)
change = True
if change:
search_for(search)
return redirect(url_for('listings.landing', search=search.pk, page=0))