def treat():
global final_html
global df,origin_df
chi_key = list()
firstkey = False
secondkey = False
if request.method == 'POST':
Listkey1 = list(MultiDict(request.form).values())
Listkey2 = MultiDict(request.form)
df1 = df
for key1 in Listkey1:
if(key1 == "Replace nulls,NAs with zeros"):
firstkey = True
secondkey = False
elif(key1 == "Custom: Enter values to replace"):
secondkey = True
firstkey = False
if(secondkey):
replace_str = str(Listkey2.get('changeval'))
for key1 in Listkey1:
if(key1 <> "Replace nulls,NAs with zeros" and key1 <> replace_str and key1 <> "Custom: Enter values to replace"):
df[key1].fillna(replace_str)
elif(firstkey):
replace_str = str(Listkey2.get('changeval'))
for key1 in Listkey1:
if(key1 <> "Replace nulls,NAs with zeros" and key1 <> replace_str and key1 <> "Custom: Enter values to replace"):
df[key1].fillna(0)
temp_df = df[1:15]
final_html = template.s1 + "</div><br>Null values were removed<br><br>" + temp_df.to_html()
return final_html
return 'helloo'
def url_for_other_page(page, anchor=None):
"""
Generates a URL for the same page, with the only difference
being that a new "page" query string value is updated
with the provided page number.
Args:
page: The new page number to use.
anchor: The anchor to use in the URL. Optional.
Returns:
The URL for the current page with a new page number.
"""
# Set up our args based on query string arguments
args = MultiDict(request.args)
# Now merge in request.view_args to prevent strange query string
# overrides of these values
args.update(request.view_args)
# Explicitly override the page
args['page'] = page
# We have to explicitly pass in the anchor here -
# otherwise, it gets interpreted as a list with a single
# element, which breaks the URL generation
# (#['anchor'] versus #anchor)
if anchor is not None and len(anchor) > 0:
return url_for(request.endpoint, _anchor=anchor, **args)
else:
return url_for(request.endpoint, **args)
def tree2():
global final_html
global df,origin_df
chi_key = list()
firstkey = ""
init_style_string = """<p style="position: absolute; font-size: 12px; top: <top>px; width: <width>px; height: <height>px; left:<left>px; text-align: center;">tree_text_here</p>"""
if request.method == 'POST':
Listkey1 = list(MultiDict(request.form).values())
Listkey2 = MultiDict(request.form)
DV_tree = Listkey2.get('DV')
df1 = df
for key1 in Listkey1:
if(key1 <> "Build Tree" and key1 <> DV_tree):
chi_key.append(key1)
df1 = df.loc[:,chi_key]
df2 = df1.values
temp_count = 0
Y = df[DV_tree]
clf = tree.DecisionTreeClassifier()
clf = clf.fit(df2,Y.values)
dot_data = StringIO()
tree.export_graphviz(clf, out_file=dot_data)
k = dot_data.getvalue()
k1 = k.split(";")
left_px = 600
width_px = 150
top_px = 50
height_px = 309
s = build_tree_html(k,init_style_string,left_px,width_px,top_px,height_px)
temp_df = df[0:15]
t = """</div><div style="float:right;"><br> Decision Tree result <br>"""
final_html = template.s1 + t + k + "</div><br><br><br>" + temp_df.to_html()
return final_html
return 'helloo'
def _formitem(self):
# This is the WTForm to validate input data.
mform = getattr(self.model, "__form__", None)
# Flask-WTForms requires the data to be a MultiDict instance.
# request.form is already MultiDict but request.json is not.
data = MultiDict(request.json) if request.json is not None else request.form
if mform is not None:
form = mform(data, csrf_enabled=False)
if not form.validate():
return None, form.errors
data = strip_nonitems(form.data, data)
try:
item = self.model()
if hasattr(item, "update_raw"):
err = item.update_raw(data)
else:
err = {}
for k, v in data.iteritems():
func = getattr(item, "update_" + k, None)
if func and callable(func):
e = func(v)
if e:
err[k] = e
else:
item[k] = v
if not err:
err = None
except (AssertionError, AttributeError, ValueError) as e:
return None, e.message
return item, err
def type_convert(self, obj):
if obj is None:
return None
if isinstance(obj, (dict, list)) and not isinstance(obj, MultiDict):
return obj
if isinstance(obj, Headers):
obj = MultiDict(obj.iteritems())
result = dict()
convert_funs = {
'integer': lambda v: int(v[0]),
'boolean': lambda v: v[0].lower() not in ['n', 'no', 'false', '', '0'],
'null': lambda v: None,
'number': lambda v: float(v[0]),
'string': lambda v: v[0]
}
def convert_array(type_, v):
func = convert_funs.get(type_, lambda v: v[0])
return [func([i]) for i in v]
for k, values in obj.iterlists():
prop = self.validator.schema['properties'].get(k, {})
type_ = prop.get('type')
fun = convert_funs.get(type_, lambda v: v[0])
if type_ == 'array':
item_type = prop.get('items', {}).get('type')
result[k] = convert_array(item_type, values)
else:
result[k] = fun(values)
return result
def parse_args(raw_args):
args = MultiDict()
for (arg_name, arg_outer_values) in raw_args.iterlists():
for arg_inner_values in arg_outer_values:
for arg_value in arg_inner_values.split(','):
args.add(arg_name, arg_value.strip())
return args
def _compute_dependency_order(skills):
"""
Add a field to each skill indicating the order it was learned
based on the skill's dependencies. Multiple skills will have the same
position if they have the same dependencies.
"""
# Key skills by first dependency. Dependency sets can be uniquely
# identified by one dependency in the set.
dependency_to_skill = MultiDict(
[(skill["dependencies_name"][0] if skill["dependencies_name"] else "", skill) for skill in skills]
)
# Start with the first skill and trace the dependency graph through
# skill, setting the order it was learned in.
index = 0
previous_skill = ""
while True:
for skill in dependency_to_skill.getlist(previous_skill):
skill["dependency_order"] = index
index += 1
# Figure out the canonical dependency for the next set of skills.
skill_names = set([skill["name"] for skill in dependency_to_skill.getlist(previous_skill)])
canonical_dependency = skill_names.intersection(set(dependency_to_skill.keys()))
if canonical_dependency:
previous_skill = canonical_dependency.pop()
else:
# Nothing depends on these skills, so we're done.
break
return skills
def cheat():
args = MultiDict()
args.update(flask.request.form)
if flask.request.method == "GET":
args.update(flask.request.args)
form = CheatForm(args)
if flask.request.method == "POST" and form.validate():
user = User.objects.with_id(form.name.data)
m = False
if form.item.data and form.quantity.data:
m = True
q = user.inventory.get(form.item.data, 0) + form.quantity.data
user.inventory[form.item.data] = q
flask.flash("Added {} item{}".format(form.quantity.data,
"s" if form.quantity.data > 1 else ""))
if form.coin.data:
m = True
user.balance += form.coin.data
flask.flash("Added {} coin{}".format(form.coin.data,
"s" if form.coin.data > 1 else ""))
if not m:
flask.flash("No changes made!","error")
else:
user.save()
return flask.redirect(flask.url_for(".single_user", name=user.name))
return flask.render_template("form.html", form=form)
def test1():
global final_html
global df,df_train,df_test,test_train_created,origin_df
unique_count = 0
err_key=0
if request.method == 'POST':
try:
Listkey1 = list(MultiDict(request.form).values())
Listkey2 = MultiDict(request.form)
test_per = int(Listkey2.get('test_percent'))
temp_df = df[1:15]
if(float(test_per) < 0 or float(test_per) > 100):
err_key = 1
if(err_key==0):
prop_test = float(test_per)/float(100)
msk = np.random.rand(len(df)) < prop_test
df_train = df[~msk]
df_test = df[msk]
print "Length of Train",len(df_train)
print "Length of Test",len(df_test)
test_train_created = True
final_html = template.s1 + "</div><br><br> Test and Train datasets created <br> " + temp_df.to_html()
elif(err_key==1):
final_html = template.s1 + """<br><br> <font color="red"> Please enter a valid percentage value </font> <br></div> """ + temp_df.to_html()
return final_html
except ValueError:
final_html = template.s1 + """<br><br><font color="lightcoral"> Error. Please enter proper value to create Test and Train dataset. </font><br> </div>""" + df[1:15].to_html()
return 'helloo'
def peek_query(args):
if not isinstance(args, MultiDict):
args = MultiDict(args)
text = args.get('q', '').strip()
q = text_query(text)
filters = parse_filters(args)
for entity in args.getlist('entity'):
filters.append(('entities.id', entity))
q = filter_query(q, filters, [])
q = add_filter(q, {
'not': {
'terms': {
'collection_id': authz.collections(authz.READ)
}
}
})
q = {
'query': q,
'size': 0,
'aggregations': {
'collections': {
'terms': {'field': 'collection_id', 'size': 30}
}
},
'_source': False
}
# import json
# print json.dumps(q, indent=2)
result = get_es().search(index=get_es_index(), body=q,
doc_type=TYPE_DOCUMENT)
aggs = result.get('aggregations', {}).get('collections', {})
buckets = aggs.get('buckets', [])
q = Collection.all_by_ids([b['key'] for b in buckets])
q = q.filter(Collection.creator_id != None) # noqa
objs = {o.id: o for o in q.all()}
roles = {}
for bucket in buckets:
collection = objs.get(bucket.get('key'))
if collection is None or collection.private:
continue
if collection.creator_id in roles:
roles[collection.creator_id]['total'] += bucket.get('doc_count')
else:
roles[collection.creator_id] = {
'name': collection.creator.name,
'email': collection.creator.email,
'total': bucket.get('doc_count')
}
roles = sorted(roles.values(), key=lambda r: r['total'], reverse=True)
roles = [format_total(r) for r in roles]
total = result.get('hits', {}).get('total')
return format_total({
'roles': roles,
'active': total > 0,
'total': total
})
def payload():
""" Performs sanity checks or decoding depending on the Content-Type,
then returns the request payload as a dict. If request Content-Type is
unsupported, aborts with a 400 (Bad Request).
.. versionchanged:: 0.7
Allow 'multipart/form-data' form fields to be JSON encoded, once the
MULTIPART_FORM_FIELDS_AS_JSON setting was been set.
.. versionchanged:: 0.3
Allow 'multipart/form-data' content type.
.. versionchanged:: 0.1.1
Payload returned as a standard python dict regardless of request content
type.
.. versionchanged:: 0.0.9
More informative error messages.
request.get_json() replaces the now deprecated request.json
.. versionchanged:: 0.0.7
Native Flask request.json preferred over json.loads.
.. versionadded: 0.0.5
"""
content_type = request.headers.get('Content-Type', '').split(';')[0]
if content_type in config.JSON_REQUEST_CONTENT_TYPES:
return request.get_json(force=True)
elif content_type == 'application/x-www-form-urlencoded':
return multidict_to_dict(request.form) if len(request.form) else \
abort(400, description='No form-urlencoded data supplied')
elif content_type == 'multipart/form-data':
# as multipart is also used for file uploads, we let an empty
# request.form go through as long as there are also files in the
# request.
if len(request.form) or len(request.files):
# merge form fields and request files, so we get a single payload
# to be validated against the resource schema.
formItems = MultiDict(request.form)
if config.MULTIPART_FORM_FIELDS_AS_JSON:
for key, lst in formItems.lists():
new_lst = []
for value in lst:
try:
new_lst.append(json.loads(value))
except ValueError:
new_lst.append(json.loads('"{0}"'.format(value)))
formItems.setlist(key, new_lst)
payload = CombinedMultiDict([formItems, request.files])
return multidict_to_dict(payload)
else:
abort(400, description='No multipart/form-data supplied')
else:
abort(400, description='Unknown or no Content-Type header supplied')
def test_multidict_pickle():
"""MultiDict types are pickle-able"""
for protocol in xrange(pickle.HIGHEST_PROTOCOL + 1):
print 'pickling protocol', protocol
d = MultiDict()
d.setlist('foo', [1, 2, 3, 4])
d.setlist('bar', 'foo bar baz'.split())
s = pickle.dumps(d, protocol)
ud = pickle.loads(s)
assert type(ud) is type(d)
print ud.lists()
assert ud == d
ud['newkey'] = 'bla'
assert ud != d
im = ImmutableMultiDict(d)
assert im == d
s = pickle.dumps(im, protocol)
ud = pickle.loads(s)
assert ud == im
assert type(ud) is type(im)
c = CombinedMultiDict([ud, im])
cc = pickle.loads(pickle.dumps(c, protocol))
assert c == cc
assert type(c) is type(cc)
def createMultiDict(file_name):
Mdict, uniqueTags = MD(), []
count = max(enumerate(open(file_name)))[0]
iterators = lineGenerator(file_name)
super_dictionary= ast.literal_eval(open("super_dictionary.json","r").readline())
bar = progressbar.ProgressBar(maxval= count, \
widgets=[progressbar.Bar(':', '[', ']'), ' ', progressbar.Percentage()])
#Variant #Context #Topic #Tag
try:
for i in xrange(count):
trans = string.maketrans('\n', '\t')# because there are \n characters on each line
line = string.translate(iterators.next(), trans).split('\t')[:-1]
lookup = super_dictionary[line[1]]
line_words = list(set(SM.getWords(line[1])).union(set(lookup[0])))
if line[0] not in uniqueTags:
uniqueTags = uniqueTags +[line[0]]
for c in xrange(len(line_words)):
Mdict.setlistdefault(line_words[c]).extend([[ line_words ,line, lookup[1], lookup[2] ]])
bar.update(i+1)
bar.finish()
except StopIteration:
pass
return Mdict, uniqueTags
def inspire_search_factory(self, search):
"""Parse query using Inspire-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
query_string = request.values.get('q', '')
urlkwargs = MultiDict()
try:
search = search.query_from_iq(query_string)
except SyntaxError:
current_app.logger.debug(
"Failed parsing query: {0}".format(
request.values.get('q', '')),
exc_info=True)
raise InvalidQueryRESTError()
search_index = search._index[0]
search, urlkwargs = inspire_filter_factory(search, urlkwargs, search_index)
search, sortkwargs = default_sorter_factory(search, search_index)
search = select_source(search)
urlkwargs.add('q', query_string)
current_app.logger.debug(json.dumps(search.to_dict(), indent=4))
return search, urlkwargs
def bin2():
global final_html
global df,origin_df
unique_count = 0
chi_key = list()
firstkey = ""
bin_val_fail = False
if request.method == 'POST':
Listkey1 = list(MultiDict(request.form).values())
Listkey2 = MultiDict(request.form)
bin_val = int(Listkey2.get('changeval'))
for key1 in Listkey1:
if(key1 <> "Bin the Columns" and key1 <> str(bin_val)):
k3 = set(df[key1])
unique_count = sum(1 for num in k3)
if(unique_count <= bin_val):
bin_val_fail = True
break
temp_bin_col = pd.qcut(df[key1],bin_val)
temp_col_name = key1 + "_bin"
df[temp_col_name] = Series(temp_bin_col.labels)
temp_df = df[1:15]
if(bin_val_fail):
final_html = template.s1 + "</div><br><br> Bin count is more than the number of categories in columns <br> " + temp_df.to_html()
else:
final_html = template.s1 + "</div><br>" + temp_df.to_html()
return final_html
return 'helloo'
def tree3():
global final_html
global df,df_train,df_test,test_train_created,origin_df
chi_key = list()
init_style_string = template.style_string
if request.method == 'POST':
Listkey1 = list(MultiDict(request.form).values())
Listkey2 = MultiDict(request.form)
DV_tree = Listkey2.get('DV')
df1 = df
for key1 in Listkey1:
if(key1 <> "Build Tree" and key1 <> DV_tree):
chi_key.append(key1)
df1 = df.loc[:,chi_key]
df2 = df1.values
Y = df[DV_tree]
clf = tree.DecisionTreeClassifier()
clf = clf.fit(df2,Y.values)
dot_data = StringIO()
tree.export_graphviz(clf, out_file=dot_data)
k = dot_data.getvalue()
left_px = 600
width_px = 150
top_px = 50
height_px = 309
s = build_tree_html(k,init_style_string,left_px,width_px,top_px,height_px)
temp_df = df[0:15]
t = """</div><div style="width:600px; height:700px; position: absolute; top: 20px; left:500px;"><br> Decision Tree result <br>"""
final_html = template.s1 + t + k + "<br><br></div>" + temp_df.to_html()
return final_html
return 'helloo'
def get_url(self, **kwargs):
"""Returns the URL for the viz"""
d = self.orig_form_data.copy()
if 'json' in d:
del d['json']
if 'action' in d:
del d['action']
d.update(kwargs)
# Remove unchecked checkboxes because HTML is weird like that
od = MultiDict()
for key in sorted(d.keys()):
if d[key] is False:
del d[key]
else:
if isinstance(d, MultiDict):
v = d.getlist(key)
else:
v = d.get(key)
if not isinstance(v, list):
v = [v]
for item in sorted(v):
od.add(key, item)
href = Href(
'/caravel/explore/{self.datasource.type}/'
'{self.datasource.id}/'.format(**locals()))
return href(od)
def treat():
global final_html
global df,df_train,df_test,test_train_created,origin_df
firstkey = False
secondkey = False
if request.method == 'POST':
try:
Listkey1 = list(MultiDict(request.form).values())
Listkey2 = MultiDict(request.form)
for key1 in Listkey1:
if(key1 == "Replace nulls,NAs with zeros"):
firstkey = True
secondkey = False
elif(key1 == "Custom: Enter values to replace"):
secondkey = True
firstkey = False
if(secondkey):
replace_str = str(Listkey2.get('changeval'))
for key1 in Listkey1:
if(key1 <> "Replace nulls,NAs with zeros" and key1 <> replace_str and key1 <> "Custom: Enter values to replace"):
df[key1].fillna(replace_str)
elif(firstkey):
replace_str = str(Listkey2.get('changeval'))
for key1 in Listkey1:
if(key1 <> "Replace nulls,NAs with zeros" and key1 <> replace_str and key1 <> "Custom: Enter values to replace"):
df[key1].fillna(0)
temp_df = df[1:15]
final_html = template.s1 + "<br>Null values were replaced!!<br><br></div>" + temp_df.to_html()
return final_html
except:
final_html = template.s1 + """<br><font color="lightcoral"> Something went wrong. Please try again !! <font><br><br></div>""" + df[1:15].to_html()
return final_html
return 'helloo'
def get_input_from_xml(doc):
the_input = MultiDict()
for input_el in xpath_ns(doc, '/wps:Execute/wps:DataInputs/wps:Input'):
[identifier_el] = xpath_ns(input_el, './ows:Identifier')
[value_el] = xpath_ns(input_el, './wps:Data/wps:LiteralData')
the_input.update({identifier_el.text: text_type(value_el.text)})
return the_input
def get_form(self, formdata=None, load_draft=True,
validate_draft=False):
"""
Create form instance with draft data and form data if provided.
:param formdata: Incoming form data.
:param files: Files to ingest into form
:param load_draft: True to initialize form with draft data.
:param validate_draft: Set to true to validate draft data, when no form
data is provided.
"""
if not self.has_form():
raise FormDoesNotExists(self.id)
draft_data = unicodifier(self.values) if load_draft else {}
formdata = MultiDict(formdata or {})
form = self.form_class(
formdata=formdata, **draft_data
)
if formdata:
form.reset_field_data(exclude=formdata.keys())
# Set field flags
if load_draft and self.flags:
form.set_flags(self.flags)
if validate_draft and draft_data and formdata is None:
form.validate()
return form
def strip_sensitive_data_from_multidict(multidict):
"""
Creates a new MultiDict with sensitive data blotted out for logging purposes. Tightly coupled to the Flask
logger and werkzeug.datastructures classes, we might want to make this more generic in the future.
:param multidict: Multidict
:return: Dictionary of keys and values (potentially stripped)
"""
from werkzeug.datastructures import MultiDict
# A bit overkill for now, since we have so few exceptions, and I'd hate for something to actually sneak through
sensitive_regex = [
r'.*account.*',
r'.*bank.*',
r'.*password.*',
r'.*routing.*'
]
stripped_multidict = MultiDict()
# We perform an iterlists so that we get duplicates of the same key.
for key, value in multidict.iterlists():
for regex in sensitive_regex:
match = re.match(regex, key, re.DOTALL | re.IGNORECASE)
if match:
value = u'************'
break
stripped_multidict.add(key, value)
return stripped_multidict
def register():
data = MultiDict(mapping=request.json)
inputs = RegistrationForm(data, csrf_enabled=False)
if not inputs.validate():
return transform(300, message="Invalid inputs")
else:
firstName = data.get('first_name')
lastName = data.get('last_name')
email = data.get('email')
password = data.get('password')
user = User(email, password, firstName, lastName)
auth = AuthToken()
user.tokens.append(auth)
try:
db.session.add(user)
# db.session.add(auth)
db.session.commit()
except IntegrityError as e:
return jsonify({"error": "email already taken"})
response = auth.__repr__()
response.update({
'user_id': user.id,
'first_name': user.first_name,
'last_name': user.last_name
})
return jsonify(response)
def createMultiDict(file_name):
Mdict, uniqueTags = MD(), []
count = max(enumerate(open(file_name)))[0]
iterators = lineGenerator(file_name)
try:
for i in xrange(count):
trans = string.maketrans('\n', '\t')# because there are \n characters on each line
line = string.translate(iterators.next(), trans).split('\t')
tag, entity = line[0], line[1]
if tag not in uniqueTags:
uniqueTags = uniqueTags +[tag]
entity = getWords(entity) # Extract words from sentence: Stopwords removed, punctuations removed
# remove ' ' and "" from the list
entity = [i for i in entity if entity!="" or entity!=" "]
line_words = entity + [tag] # Words in a single line of the file
for c in xrange(len(line_words)-1):
Mdict.setlistdefault(line_words[c]).extend([line_words[c+1:] + line_words[:c] + [line_words[c]]])
except StopIteration:
pass
return Mdict, uniqueTags
def linreg2():
global final_html
global df,origin_df
lin_reg_key = list()
firstkey = ""
if request.method == 'POST':
Listkey1 = list(MultiDict(request.form).values())
Listkey2 = MultiDict(request.form)
DV_lin_reg = Listkey2.get('DV')
df1 = df
for key1 in Listkey1:
if(key1 <> "Build Linear Regression Model" and key1 <> DV_lin_reg):
lin_reg_key.append(key1)
df1 = df.loc[:,lin_reg_key]
df2 = df1.values
temp_count = 0
Y = df[DV_lin_reg]
linreg = linear_model.LinearRegression()
fit1 = linreg.fit(df2,Y.values)
s = fit1.coef_
temp_df = df[0:15]
t = """</div><div style="float:right;"><br> Linear Regression Result <br>"""
final_html = template.s1 + t + s + "</div><br><br><br>" + temp_df.to_html()
return final_html
return 'helloo'
def reset_password():
if current_user.is_authenticated():
logout_user()
form = ResetPasswordForm()
if form.is_submitted():
if form.validate_on_submit():
try:
user = auth_service.user_by_username_or_email(form.email.data)
except NoResultFound:
form.email.errors = ['User with this email doesn\'t exist.']
else:
if user.active:
profile = profiles_service.profile(user.id)
auth_service.send_password_reset_mail(profile)
flash('We sent a letter that will allow you to change your password. Please check your mailbox.', 'success')
return redirect(url_for('auth.login'))
else:
form.email.errors = ['This email address is not confirmed. Please, confirm it first.']
else:
initial = MultiDict()
email = request.values.get('email')
if email:
initial.add('email', email)
form = ResetPasswordForm(formdata=initial)
return render_template('auth/reset_password.html',
reset_password_form=form)
def from_params(params: MultiDict, items: int) -> 'Page':
# page, page-size, limit (deprecated)
page = params.get('page', 1, int)
limit = params.get('limit', 0, int)
page_size = params.get('page-size', limit, int)
return Page(page, page_size, items)
def get_available_products():
'''
Search with given arguments,
but also limit the minimum amount to at least 1.
'''
args = MultiDict(request.args)
args.add('minamount', 1)
return dumps(backend.get_products(args))
def test_should_strip_page_from_multidict():
params = MultiDict()
params.add("this", "that")
params.add("page", 100)
parsed = search_helpers.query_args_for_pagination(params)
assert_equal(parsed['this'], 'that')
assert_false('page' in parsed)
def delete(self, itemid):
data = MultiDict(request.json) if request.json is not None else request.form
if data:
data = data.to_dict()
filt = self.filter_by.copy() if self.filter_by else {}
filt["_id"] = itemid
self.model.update(filt, {"$pull": {self.field: data}})
return jsonify(success=True)
def get_by_category(category):
'''
Search with given arguments,
but also limit the search to the given category.
'''
args = MultiDict(request.args)
args.add('category', category)
return dumps(backend.get_products(args))
from werkzeug.datastructures import MultiDict
post = MultiDict()
post.add("question", "answer")
def item_path_search_factory(self, search, index_id=None):
"""Parse query using Weko-Query-Parser.
:param self: REST view.
:param search: Elastic search DSL search instance.
:returns: Tuple with search instance and URL arguments.
"""
def _get_index_earch_query():
query_q = {
"_source": {
"excludes": ['content']
},
"query": {
"bool": {
"must": [{
"match": {
"path.tree": "@index"
}
}, {
"match": {
"relation_version_is_last": "true"
}
}]
}
},
"aggs": {
"path": {
"terms": {
"field": "path.tree",
"include": "@index|@index/[^/]+",
"size": "@count"
},
"aggs": {
"date_range": {
"filter": {
"match": {
"publish_status": "0"
}
},
"aggs": {
"available": {
"range": {
"field":
"publish_date",
"ranges": [{
"from": "now+1d/d"
}, {
"to": "now+1d/d"
}]
},
}
}
},
"no_available": {
"filter": {
"bool": {
"must_not": [{
"match": {
"publish_status": "0"
}
}]
}
}
}
}
}
},
"post_filter": {}
}
# add item type aggs
query_q['aggs']['path']['aggs']. \
update(get_item_type_aggs(search._index[0]))
q = request.values.get('q') if index_id is None else index_id
if q:
mut = get_permission_filter(q)
else:
mut = get_permission_filter()
if mut:
mut = list(map(lambda x: x.to_dict(), mut))
post_filter = query_q['post_filter']
if mut[0].get('bool'):
post_filter['bool'] = mut[0]['bool']
else:
post_filter['bool'] = {'must': mut}
# create search query
if q:
try:
fp = Indexes.get_self_path(q)
if fp:
query_q = json.dumps(query_q).replace("@index", fp.path)
query_q = json.loads(query_q)
except BaseException:
pass
query_q = json.dumps(query_q).replace("@count",
str(Indexes.get_index_count()))
query_q = json.loads(query_q)
return query_q
# create a index search query
query_q = _get_index_earch_query()
urlkwargs = MultiDict()
try:
# Aggregations.
extr = search._extra.copy()
search.update_from_dict(query_q)
search._extra.update(extr)
except SyntaxError:
q = request.values.get('q', '') if index_id is None else index_id
current_app.logger.debug("Failed parsing query: {0}".format(q),
exc_info=True)
raise InvalidQueryRESTError()
from invenio_records_rest.sorter import default_sorter_factory
search_index = search._index[0]
search, sortkwargs = default_sorter_factory(search, search_index)
for key, value in sortkwargs.items():
# set custom sort option
if 'custom_sort' in value:
ind_id = request.values.get('q', '')
search._sort = []
if value == 'custom_sort':
script_str, default_sort = SearchSetting.get_custom_sort(
ind_id, 'asc')
else:
script_str, default_sort = SearchSetting.get_custom_sort(
ind_id, 'desc')
search._sort.append(script_str)
search._sort.append(default_sort)
# set selectbox
urlkwargs.add(key, value)
# default sort
if not sortkwargs:
sort_key, sort = SearchSetting.get_default_sort(
current_app.config['WEKO_SEARCH_TYPE_INDEX'])
sort_obj = dict()
key_fileds = SearchSetting.get_sort_key(sort_key)
if 'custom_sort' not in sort_key:
if sort == 'desc':
sort_obj[key_fileds] = dict(order='desc')
sort_key = '-' + sort_key
else:
sort_obj[key_fileds] = dict(order='asc')
search._sort.append(sort_obj)
else:
if sort == 'desc':
ind_id = request.values.get('q', '')
script_str, default_sort = SearchSetting.get_custom_sort(
ind_id, 'desc')
sort_key = '-' + sort_key
else:
script_str, default_sort = SearchSetting.get_custom_sort(
ind_id, 'asc')
search._sort = []
search._sort.append(script_str)
search._sort.append(default_sort)
urlkwargs.add('sort', sort_key)
urlkwargs.add('q', query_q)
return search, urlkwargs
def reset(self):
data = MultiDict([('csrf', self.generate_csrf_token())])
self.process(data)
def search_page(product_name: str = None,
year: int = None,
month: int = None,
day: int = None):
(
product,
product_summary,
selected_summary,
year_selector_summary,
time_selector_summary,
) = _load_product(product_name, year, month, day)
time_range = utils.as_time_range(year,
month,
day,
tzinfo=_model.STORE.grouping_timezone)
args = MultiDict(flask.request.args)
query = utils.query_to_search(args, product=product)
# Always add time range, selected product to query
if product_name:
query["product"] = product_name
if "time" in query:
# If they left one end of the range open, fill it in with the product bounds.
search_time = query["time"]
assert isinstance(search_time, Range)
if product_summary:
query["time"] = Range(
search_time.begin or product_summary.time_earliest,
search_time.end
or product_summary.time_latest + timedelta(days=1),
)
# The URL time range always trumps args.
if time_range:
query["time"] = time_range
_LOG.info("query", query=query)
# TODO: Add sort option to index API
datasets = sorted(
_model.STORE.index.datasets.search(**query,
limit=_HARD_SEARCH_LIMIT + 1),
key=lambda d: d.center_time,
)
more_datasets_exist = False
if len(datasets) > _HARD_SEARCH_LIMIT:
more_datasets_exist = True
datasets = datasets[:_HARD_SEARCH_LIMIT]
if request_wants_json():
return as_rich_json(
dict(datasets=[
build_dataset_info(_model.STORE.index, d) for d in datasets
]))
# For display on the page (and future searches).
if "time" not in query and product_summary and product_summary.time_earliest:
query["time"] = Range(
product_summary.time_earliest,
product_summary.time_latest + timedelta(days=1),
)
return utils.render(
"search.html",
year=year,
month=month,
day=day,
product=product,
# Summary for the whole product
product_summary=product_summary,
# Summary for the users' currently selected filters.
selected_summary=selected_summary,
datasets=datasets,
query_params=query,
there_are_more_results=more_datasets_exist,
time_selector_summary=time_selector_summary,
year_selector_summary=year_selector_summary,
)
def test_registration_form_username_exists(init_database, app):
user = User.create('username1', 'username1')
with app.test_request_context('/'):
form = RegistrationForm(MultiDict(registraition_data))
assert form.validate() == False
assert 'username exists' in form.errors['username']
def test_patron_auth_services_post_errors(self):
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("protocol", "Unknown"),
])
response = (self.manager.admin_patron_auth_services_controller.
process_patron_auth_services())
assert response == UNKNOWN_PROTOCOL
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([])
response = (self.manager.admin_patron_auth_services_controller.
process_patron_auth_services())
assert response == NO_PROTOCOL_FOR_NEW_SERVICE
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("id", "123"),
])
response = (self.manager.admin_patron_auth_services_controller.
process_patron_auth_services())
assert response == MISSING_SERVICE
auth_service, ignore = create(
self._db,
ExternalIntegration,
protocol=SimpleAuthenticationProvider.__module__,
goal=ExternalIntegration.PATRON_AUTH_GOAL,
name="name",
)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("id", auth_service.id),
("protocol", SIP2AuthenticationProvider.__module__),
])
response = (self.manager.admin_patron_auth_services_controller.
process_patron_auth_services())
assert response == CANNOT_CHANGE_PROTOCOL
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("name", auth_service.name),
("protocol", SIP2AuthenticationProvider.__module__),
])
response = (self.manager.admin_patron_auth_services_controller.
process_patron_auth_services())
assert response == INTEGRATION_NAME_ALREADY_IN_USE
auth_service, ignore = create(
self._db,
ExternalIntegration,
protocol=MilleniumPatronAPI.__module__,
goal=ExternalIntegration.PATRON_AUTH_GOAL,
)
common_args = self._common_basic_auth_arguments()
with self.request_context_with_admin("/", method="POST"):
M = MilleniumPatronAPI
flask.request.form = MultiDict([
("name", "some auth name"),
("id", auth_service.id),
("protocol", MilleniumPatronAPI.__module__),
(ExternalIntegration.URL, "http://url"),
(M.AUTHENTICATION_MODE, "Invalid mode"),
(M.VERIFY_CERTIFICATE, "true"),
] + common_args)
response = (self.manager.admin_patron_auth_services_controller.
process_patron_auth_services())
assert response.uri == INVALID_CONFIGURATION_OPTION.uri
auth_service, ignore = create(
self._db,
ExternalIntegration,
protocol=SimpleAuthenticationProvider.__module__,
goal=ExternalIntegration.PATRON_AUTH_GOAL,
)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("id", auth_service.id),
("protocol", SimpleAuthenticationProvider.__module__),
])
response = (self.manager.admin_patron_auth_services_controller.
process_patron_auth_services())
assert response.uri == INCOMPLETE_CONFIGURATION.uri
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("protocol", SimpleAuthenticationProvider.__module__),
("libraries", json.dumps([{
"short_name": "not-a-library"
}])),
] + common_args)
response = (self.manager.admin_patron_auth_services_controller.
process_patron_auth_services())
assert response.uri == NO_SUCH_LIBRARY.uri
library, ignore = create(
self._db,
Library,
name="Library",
short_name="L",
)
auth_service.libraries += [library]
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("protocol", SimpleAuthenticationProvider.__module__),
(
"libraries",
json.dumps([{
"short_name":
library.short_name,
AuthenticationProvider.LIBRARY_IDENTIFIER_RESTRICTION_TYPE:
AuthenticationProvider.
LIBRARY_IDENTIFIER_RESTRICTION_TYPE_NONE,
AuthenticationProvider.LIBRARY_IDENTIFIER_FIELD:
AuthenticationProvider.
LIBRARY_IDENTIFIER_RESTRICTION_BARCODE,
}]),
),
] + common_args)
response = (self.manager.admin_patron_auth_services_controller.
process_patron_auth_services())
assert response.uri == MULTIPLE_BASIC_AUTH_SERVICES.uri
self._db.delete(auth_service)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("protocol", SimpleAuthenticationProvider.__module__),
(
"libraries",
json.dumps([{
"short_name":
library.short_name,
AuthenticationProvider.LIBRARY_IDENTIFIER_RESTRICTION_TYPE:
AuthenticationProvider.
LIBRARY_IDENTIFIER_RESTRICTION_TYPE_NONE,
AuthenticationProvider.LIBRARY_IDENTIFIER_FIELD:
AuthenticationProvider.
LIBRARY_IDENTIFIER_RESTRICTION_BARCODE,
AuthenticationProvider.EXTERNAL_TYPE_REGULAR_EXPRESSION:
"(invalid re",
}]),
),
] + common_args)
response = (self.manager.admin_patron_auth_services_controller.
process_patron_auth_services())
assert response == INVALID_EXTERNAL_TYPE_REGULAR_EXPRESSION
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("protocol", SimpleAuthenticationProvider.__module__),
(
"libraries",
json.dumps([{
"short_name":
library.short_name,
AuthenticationProvider.LIBRARY_IDENTIFIER_RESTRICTION_TYPE:
AuthenticationProvider.
LIBRARY_IDENTIFIER_RESTRICTION_TYPE_REGEX,
AuthenticationProvider.LIBRARY_IDENTIFIER_FIELD:
AuthenticationProvider.
LIBRARY_IDENTIFIER_RESTRICTION_BARCODE,
AuthenticationProvider.LIBRARY_IDENTIFIER_RESTRICTION:
"(invalid re",
}]),
),
] + common_args)
response = (self.manager.admin_patron_auth_services_controller.
process_patron_auth_services())
assert response == INVALID_LIBRARY_IDENTIFIER_RESTRICTION_REGULAR_EXPRESSION
self.admin.remove_role(AdminRole.SYSTEM_ADMIN)
self._db.flush()
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("protocol", SimpleAuthenticationProvider.__module__),
] + self._common_basic_auth_arguments())
pytest.raises(
AdminNotAuthorized,
self.manager.admin_patron_auth_services_controller.
process_patron_auth_services,
)
def test_repeating_group_no_skip(self):
# Given I add some people
form_data = MultiDict()
form_data.add('household-0-first-name', 'Joe')
form_data.add('household-0-middle-names', '')
form_data.add('household-0-last-name', 'Bloggs')
form_data.add('household-1-first-name', 'Jane')
form_data.add('household-1-middle-names', '')
form_data.add('household-1-last-name', 'Doe')
self.post(form_data)
self.assertInPage('Is that everyone?')
self.post({'everyone-at-address-confirmation-answer': 'Yes'})
# Then provide details for each member
joe_dob = {
'date-of-birth-answer-day': '12',
'date-of-birth-answer-month': '3',
'date-of-birth-answer-year': '1990'
}
self.post(joe_dob)
self.post({'sex-answer': 'Male'})
self.post(action='save_continue')
jane_dob = {
'date-of-birth-answer-day': '27',
'date-of-birth-answer-month': '11',
'date-of-birth-answer-year': '1995'
}
self.post(jane_dob)
self.post({'sex-answer': 'Female'})
self.post(action='save_continue')
# The survey answers should still contain the first occupant's DoB
result = self.dumpAnswers()
dobs = [
a for a in result['answers']
if a['answer_id'] == 'date-of-birth-answer'
]
self.assertEqual(len(dobs), 2, 'There should be two date-of-birth '
'answers')
self.assertEqual(dobs[0]['value'], '1990-03-12')
self.assertEqual(dobs[1]['value'], '1995-11-27')
def test_admin_auth_services_post_errors(self):
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("protocol", "Unknown"),
])
response = (self.manager.admin_auth_services_controller.
process_admin_auth_services())
assert response == UNKNOWN_PROTOCOL
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([])
response = (self.manager.admin_auth_services_controller.
process_admin_auth_services())
assert response == NO_PROTOCOL_FOR_NEW_SERVICE
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("id", "1234"),
])
response = (self.manager.admin_auth_services_controller.
process_admin_auth_services())
assert response == MISSING_SERVICE
auth_service, ignore = create(
self._db,
ExternalIntegration,
protocol=ExternalIntegration.GOOGLE_OAUTH,
goal=ExternalIntegration.ADMIN_AUTH_GOAL,
)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("id", str(auth_service.id)),
])
response = (self.manager.admin_auth_services_controller.
process_admin_auth_services())
assert response == CANNOT_CHANGE_PROTOCOL
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("protocol", "Google OAuth"),
])
response = (self.manager.admin_auth_services_controller.
process_admin_auth_services())
assert response.uri == INCOMPLETE_CONFIGURATION.uri
self.admin.remove_role(AdminRole.SYSTEM_ADMIN)
self._db.flush()
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("name", "oauth"),
("protocol", "Google OAuth"),
("url", "url"),
("username", "username"),
("password", "password"),
("domains", "nypl.org"),
])
pytest.raises(
AdminNotAuthorized,
self.manager.admin_auth_services_controller.
process_admin_auth_services,
)
def test_bad_classification(self, mock_index):
"""Request with nonsense as primary classification."""
params = MultiDict({"primary_classification": "nonsense"})
with self.assertRaises(BadRequest):
api.search(params)
def test_model_form(self):
with self.app.test_request_context("/"):
db = self.db
class BlogPost(db.Document):
meta = {"allow_inheritance": True}
title = db.StringField(required=True, max_length=200)
posted = db.DateTimeField(default=datetime.datetime.now)
tags = db.ListField(db.StringField())
class TextPost(BlogPost):
email = db.EmailField(required=False)
lead_paragraph = db.StringField(max_length=200)
content = db.StringField(required=True)
class LinkPost(BlogPost):
url = db.StringField(required=True, max_length=200)
interest = db.DecimalField(required=True)
# Create a text-based post
TextPostForm = model_form(
TextPost, field_args={"lead_paragraph": {
"textarea": True
}})
form = TextPostForm(
MultiDict({
"title": "Using MongoEngine",
"tags": ["mongodb", "mongoengine"]
}))
self.assertFalse(form.validate())
form = TextPostForm(
MultiDict({
"title": "Using MongoEngine",
"content": "See the tutorial",
"tags": ["mongodb", "mongoengine"],
}))
self.assertTrue(form.validate())
form.save()
self.assertEqual(form.title.type, "StringField")
self.assertEqual(form.content.type, "TextAreaField")
self.assertEqual(form.lead_paragraph.type, "TextAreaField")
self.assertEqual(BlogPost.objects.first().title,
"Using MongoEngine")
self.assertEqual(BlogPost.objects.count(), 1)
form = TextPostForm(
MultiDict({
"title": "Using Flask-MongoEngine",
"content": "See the tutorial",
"tags": ["flask", "mongodb", "mongoengine"],
}))
self.assertTrue(form.validate())
form.save()
self.assertEqual(BlogPost.objects.count(), 2)
post = BlogPost.objects(title="Using Flask-MongoEngine").get()
form = TextPostForm(
MultiDict({
"title": "Using Flask-MongoEngine",
"content": "See the tutorial",
"tags-0": "flask",
"tags-1": "mongodb",
"tags-2": "mongoengine",
"tags-3": "flask-mongoengine",
}),
instance=post,
)
self.assertTrue(form.validate())
form.save()
post = post.reload()
self.assertEqual(
post.tags,
["flask", "mongodb", "mongoengine", "flask-mongoengine"])
# Create a link post
LinkPostForm = model_form(LinkPost)
form = LinkPostForm(
MultiDict({
"title": "Using Flask-MongoEngine",
"url": "http://flask-mongoengine.org",
"interest": "0",
}))
form.validate()
self.assertTrue(form.validate())
def test_patron_auth_services_post_create(self):
mock_controller = self._get_mock()
library, ignore = create(
self._db,
Library,
name="Library",
short_name="L",
)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("protocol", SimpleAuthenticationProvider.__module__),
(
"libraries",
json.dumps([{
"short_name":
library.short_name,
AuthenticationProvider.EXTERNAL_TYPE_REGULAR_EXPRESSION:
"^(.)",
AuthenticationProvider.LIBRARY_IDENTIFIER_RESTRICTION_TYPE:
AuthenticationProvider.
LIBRARY_IDENTIFIER_RESTRICTION_TYPE_REGEX,
AuthenticationProvider.LIBRARY_IDENTIFIER_FIELD:
AuthenticationProvider.
LIBRARY_IDENTIFIER_RESTRICTION_BARCODE,
AuthenticationProvider.LIBRARY_IDENTIFIER_RESTRICTION:
"^1234",
}]),
),
] + self._common_basic_auth_arguments())
response = mock_controller.process_patron_auth_services()
assert response.status_code == 201
assert mock_controller.validate_formats_call_count == 1
auth_service = get_one(self._db,
ExternalIntegration,
goal=ExternalIntegration.PATRON_AUTH_GOAL)
assert auth_service.id == int(response.response[0])
assert SimpleAuthenticationProvider.__module__ == auth_service.protocol
assert ("user" == auth_service.setting(
BasicAuthenticationProvider.TEST_IDENTIFIER).value)
assert ("pass" == auth_service.setting(
BasicAuthenticationProvider.TEST_PASSWORD).value)
assert [library] == auth_service.libraries
assert (
"^(.)" == ConfigurationSetting.for_library_and_externalintegration(
self._db,
AuthenticationProvider.EXTERNAL_TYPE_REGULAR_EXPRESSION,
library,
auth_service,
).value)
common_args = self._common_basic_auth_arguments()
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("protocol", MilleniumPatronAPI.__module__),
(ExternalIntegration.URL, "url"),
(MilleniumPatronAPI.VERIFY_CERTIFICATE, "true"),
(
MilleniumPatronAPI.AUTHENTICATION_MODE,
MilleniumPatronAPI.PIN_AUTHENTICATION_MODE,
),
] + common_args)
response = mock_controller.process_patron_auth_services()
assert response.status_code == 201
assert mock_controller.validate_formats_call_count == 2
auth_service2 = get_one(
self._db,
ExternalIntegration,
goal=ExternalIntegration.PATRON_AUTH_GOAL,
protocol=MilleniumPatronAPI.__module__,
)
assert auth_service2 != auth_service
assert auth_service2.id == int(response.response[0])
assert "url" == auth_service2.url
assert ("user" == auth_service2.setting(
BasicAuthenticationProvider.TEST_IDENTIFIER).value)
assert ("pass" == auth_service2.setting(
BasicAuthenticationProvider.TEST_PASSWORD).value)
assert ("true" == auth_service2.setting(
MilleniumPatronAPI.VERIFY_CERTIFICATE).value)
assert (MilleniumPatronAPI.PIN_AUTHENTICATION_MODE ==
auth_service2.setting(
MilleniumPatronAPI.AUTHENTICATION_MODE).value)
assert None == auth_service2.setting(
MilleniumPatronAPI.BLOCK_TYPES).value
assert [] == auth_service2.libraries
def test_libraries_post_edit(self):
# A library already exists.
library = self._library("New York Public Library", "nypl")
ConfigurationSetting.for_library(Configuration.FEATURED_LANE_SIZE,
library).value = 5
ConfigurationSetting.for_library(
Configuration.DEFAULT_FACET_KEY_PREFIX +
FacetConstants.ORDER_FACET_GROUP_NAME,
library).value = FacetConstants.ORDER_RANDOM
ConfigurationSetting.for_library(
Configuration.ENABLED_FACETS_KEY_PREFIX +
FacetConstants.ORDER_FACET_GROUP_NAME, library).value = json.dumps(
[FacetConstants.ORDER_TITLE, FacetConstants.ORDER_RANDOM])
ConfigurationSetting.for_library(Configuration.LOGO,
library).value = "A tiny image"
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("uuid", library.uuid),
("name", "The New York Public Library"),
("short_name", "nypl"),
(Configuration.FEATURED_LANE_SIZE, "20"),
(Configuration.MINIMUM_FEATURED_QUALITY, "0.9"),
(Configuration.WEBSITE_URL, "https://library.library/"),
(Configuration.DEFAULT_NOTIFICATION_EMAIL_ADDRESS,
"*****@*****.**"),
(Configuration.HELP_EMAIL, "*****@*****.**"),
(Configuration.DEFAULT_FACET_KEY_PREFIX +
FacetConstants.ORDER_FACET_GROUP_NAME,
FacetConstants.ORDER_AUTHOR),
(Configuration.ENABLED_FACETS_KEY_PREFIX +
FacetConstants.ORDER_FACET_GROUP_NAME + "_" +
FacetConstants.ORDER_AUTHOR, ''),
(Configuration.ENABLED_FACETS_KEY_PREFIX +
FacetConstants.ORDER_FACET_GROUP_NAME + "_" +
FacetConstants.ORDER_RANDOM, ''),
])
flask.request.files = MultiDict([])
response = self.manager.admin_library_settings_controller.process_post(
)
eq_(response.status_code, 200)
library = get_one(self._db, Library, uuid=library.uuid)
eq_(library.uuid, response.response[0])
eq_(library.name, "The New York Public Library")
eq_(library.short_name, "nypl")
# The library-wide settings were updated.
def val(x):
return ConfigurationSetting.for_library(x, library).value
eq_("https://library.library/", val(Configuration.WEBSITE_URL))
eq_("*****@*****.**",
val(Configuration.DEFAULT_NOTIFICATION_EMAIL_ADDRESS))
eq_("*****@*****.**", val(Configuration.HELP_EMAIL))
eq_("20", val(Configuration.FEATURED_LANE_SIZE))
eq_("0.9", val(Configuration.MINIMUM_FEATURED_QUALITY))
eq_(
FacetConstants.ORDER_AUTHOR,
val(Configuration.DEFAULT_FACET_KEY_PREFIX +
FacetConstants.ORDER_FACET_GROUP_NAME))
eq_(
json.dumps(
[FacetConstants.ORDER_AUTHOR, FacetConstants.ORDER_RANDOM]),
val(Configuration.ENABLED_FACETS_KEY_PREFIX +
FacetConstants.ORDER_FACET_GROUP_NAME))
# The library-wide logo was not updated and has been left alone.
eq_(
"A tiny image",
ConfigurationSetting.for_library(Configuration.LOGO,
library).value)
def test_repeating_group_skip_second(self):
# Given I add some people
form_data = MultiDict()
form_data.add('household-0-first-name', 'Joe')
form_data.add('household-0-middle-names', '')
form_data.add('household-0-last-name', 'Bloggs')
form_data.add('household-1-first-name', 'Jane')
form_data.add('household-1-middle-names', '')
form_data.add('household-1-last-name', 'Doe')
self.post(form_data)
self.assertInPage('Is that everyone?')
self.post({'everyone-at-address-confirmation-answer': 'Yes'})
# Then provide details for the first member
joe_dob = {
'date-of-birth-answer-day': '12',
'date-of-birth-answer-month': '3',
'date-of-birth-answer-year': '1990'
}
self.post(joe_dob)
self.post({'sex-answer': 'Male'})
self.post(action='save_continue')
# When I skip DoB for second member and answer remaining questions
self.post(action='save_continue')
self.assertInPage('Is Jane Doe aged 16 or over?')
self.post({'dob-check-answer': 'Yes'})
self.assertInPage('What is Jane Doe’s sex?')
self.post({'sex-answer': 'Female'})
# The survey answers should still contain the first occupant's DoB
self.post(action='save_continue')
result = self.dumpAnswers()
joes_dob = [
a for a in result['answers']
if a['answer_id'] == 'date-of-birth-answer'
and a['group_instance'] == 0
]
self.assertEqual(
len(joes_dob), 1, 'There should be a date-of-birth '
'answer in group instance 0')
self.assertEqual(joes_dob[0]['value'], '1990-03-12')
def form(self):
d = MultiDict()
d['image'] = filedata
return d
def test_routes_based_on_answer_count(self):
""" Asserts that the routing rule is followed based on the
number of answers to a household composition question.
"""
form_data = MultiDict()
form_data.add('household-0-first-name', 'Joe')
form_data.add('household-0-middle-names', '')
form_data.add('household-0-last-name', 'Bloggs')
form_data.add('household-1-first-name', 'Jane')
form_data.add('household-1-middle-names', '')
form_data.add('household-1-last-name', 'Doe')
self.post(form_data)
self.assertInPage('This is Group 1 - you answered "2"')
def get_value(self, form: MultiDict):
return form.get(self.mongo_name, default=self.default, type=int)
def test_repeating_group_skip_first(self):
# Given I add some people
form_data = MultiDict()
form_data.add('household-0-first-name', 'Joe')
form_data.add('household-0-middle-names', '')
form_data.add('household-0-last-name', 'Bloggs')
form_data.add('household-1-first-name', 'Jane')
form_data.add('household-1-middle-names', '')
form_data.add('household-1-last-name', 'Doe')
self.post(form_data)
self.assertInPage('Is that everyone?')
self.post({'everyone-at-address-confirmation-answer': 'Yes'})
# Then provide details for each member while skipping DoB
# question for the first member
self.post(action='save_continue')
self.assertInPage('Is Joe Bloggs aged 16 or over?')
self.post({'dob-check-answer': 'Yes'})
self.post({'sex-answer': 'Male'})
self.post(action='save_continue')
jane_dob = {
'date-of-birth-answer-day': '27',
'date-of-birth-answer-month': '11',
'date-of-birth-answer-year': '1995'
}
self.post(jane_dob)
self.post({'sex-answer': 'Female'})
self.post(action='save_continue')
# The survey answers should still contain the first occupant's DoB
result = self.dumpAnswers()
janes_dob = [
a for a in result['answers']
if a['answer_id'] == 'date-of-birth-answer'
and a['group_instance'] == 1
]
self.assertEqual(
len(janes_dob), 1, 'There should be a date-of-birth '
'answer in group instance 1')
self.assertEqual(janes_dob[0]['value'], '1995-11-27')
def get_value(self, form: MultiDict):
val = form.get(self.mongo_name, "")
if len(val) == 0:
return ObjectId()
return ObjectId(val)
def test_registration_form_invalid(field, data, error, app, init_database):
with app.test_request_context('/'):
form = RegistrationForm(MultiDict(data))
assert form.validate() == False, "Field {} Error {}" % (field, error)
assert error in form.errors[field]
def _get_args(self):
if self._query_string is not None:
raise AttributeError('a query string is defined')
if self._args is None:
self._args = MultiDict()
return self._args
def test_registration_form(data, app, init_database):
with app.test_request_context('/'):
form = RegistrationForm(MultiDict(data))
assert form.validate() == True
def stream_encode_multipart(values,
use_tempfile=True,
threshold=1024 * 500,
boundary=None,
charset='utf-8'):
"""Encode a dict of values (either strings or file descriptors or
:class:`FileStorage` objects.) into a multipart encoded string stored
in a file descriptor.
"""
if boundary is None:
boundary = '---------------WerkzeugFormPart_%s%s' % (time(), random())
_closure = [BytesIO(), 0, False]
if use_tempfile:
def write_binary(string):
stream, total_length, on_disk = _closure
if on_disk:
stream.write(string)
else:
length = len(string)
if length + _closure[1] <= threshold:
stream.write(string)
else:
new_stream = TemporaryFile('wb+')
new_stream.write(stream.getvalue())
new_stream.write(string)
_closure[0] = new_stream
_closure[2] = True
_closure[1] = total_length + length
else:
write_binary = _closure[0].write
def write(string):
write_binary(string.encode(charset))
if not isinstance(values, MultiDict):
values = MultiDict(values)
for key, values in iterlists(values):
for value in values:
write('--%s\r\nContent-Disposition: form-data; name="%s"' %
(boundary, key))
reader = getattr(value, 'read', None)
if reader is not None:
filename = getattr(value, 'filename',
getattr(value, 'name', None))
content_type = getattr(value, 'content_type', None)
if content_type is None:
content_type = filename and \
mimetypes.guess_type(filename)[0] or \
'application/octet-stream'
if filename is not None:
write('; filename="%s"\r\n' % filename)
else:
write('\r\n')
write('Content-Type: %s\r\n\r\n' % content_type)
while 1:
chunk = reader(16384)
if not chunk:
break
write_binary(chunk)
else:
if not isinstance(value, string_types):
value = str(value)
else:
value = to_bytes(value, charset)
write('\r\n\r\n')
write_binary(value)
write('\r\n')
write('--%s--\r\n' % boundary)
length = int(_closure[0].tell())
_closure[0].seek(0)
return _closure[0], length, boundary
def get_value(self, form: MultiDict):
return form.getlist(self.mongo_name + "[]")
def test_libraries_post_errors(self):
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("name", "Brooklyn Public Library"),
])
response = self.manager.admin_library_settings_controller.process_post(
)
eq_(response, MISSING_LIBRARY_SHORT_NAME)
self.admin.remove_role(AdminRole.SYSTEM_ADMIN)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("name", "Brooklyn Public Library"),
("short_name", "bpl"),
])
assert_raises(
AdminNotAuthorized,
self.manager.admin_library_settings_controller.process_post)
library = self._library()
self.admin.add_role(AdminRole.LIBRARIAN, library)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("uuid", library.uuid),
("name", "Brooklyn Public Library"),
("short_name", library.short_name),
])
assert_raises(
AdminNotAuthorized,
self.manager.admin_library_settings_controller.process_post)
self.admin.add_role(AdminRole.SYSTEM_ADMIN)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = self.library_form(library, {"uuid": "1234"})
response = self.manager.admin_library_settings_controller.process_post(
)
eq_(response.uri, LIBRARY_NOT_FOUND.uri)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("name", "Brooklyn Public Library"),
("short_name", library.short_name),
])
response = self.manager.admin_library_settings_controller.process_post(
)
eq_(response, LIBRARY_SHORT_NAME_ALREADY_IN_USE)
bpl, ignore = get_one_or_create(self._db, Library, short_name="bpl")
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("uuid", bpl.uuid),
("name", "Brooklyn Public Library"),
("short_name", library.short_name),
])
response = self.manager.admin_library_settings_controller.process_post(
)
eq_(response, LIBRARY_SHORT_NAME_ALREADY_IN_USE)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("uuid", library.uuid),
("name", "The New York Public Library"),
("short_name", library.short_name),
])
response = self.manager.admin_library_settings_controller.process_post(
)
eq_(response.uri, INCOMPLETE_CONFIGURATION.uri)
# Test a web primary and secondary color that doesn't contrast
# well on white. Here primary will, secondary should not.
with self.request_context_with_admin("/", method="POST"):
flask.request.form = self.library_form(
library, {
Configuration.WEB_PRIMARY_COLOR: "#000000",
Configuration.WEB_SECONDARY_COLOR: "#e0e0e0"
})
response = self.manager.admin_library_settings_controller.process_post(
)
eq_(response.uri, INVALID_CONFIGURATION_OPTION.uri)
assert "contrast-ratio.com/#%23e0e0e0-on-%23ffffff" in response.detail
assert "contrast-ratio.com/#%23e0e0e0-on-%23ffffff" in response.detail
# Test a list of web header links and a list of labels that
# aren't the same length.
library = self._library()
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("uuid", library.uuid),
("name", "The New York Public Library"),
("short_name", library.short_name),
(Configuration.WEBSITE_URL, "https://library.library/"),
(Configuration.DEFAULT_NOTIFICATION_EMAIL_ADDRESS,
"*****@*****.**"),
(Configuration.HELP_EMAIL, "*****@*****.**"),
(Configuration.WEB_HEADER_LINKS, "http://library.com/1"),
(Configuration.WEB_HEADER_LINKS, "http://library.com/2"),
(Configuration.WEB_HEADER_LABELS, "One"),
])
response = self.manager.admin_library_settings_controller.process_post(
)
eq_(response.uri, INVALID_CONFIGURATION_OPTION.uri)
def get_value(self, form: MultiDict):
"""
Gets the value of the field from the provided form
"""
return form.get(self.mongo_name, self.default)
def form_request(data):
d = MultiDict()
d['console_logs'] = data
return d
def __init__(self,
path='/',
base_url=None,
query_string=None,
method='GET',
input_stream=None,
content_type=None,
content_length=None,
errors_stream=None,
multithread=False,
multiprocess=False,
run_once=False,
headers=None,
data=None,
environ_base=None,
environ_overrides=None,
charset='utf-8'):
path_s = make_literal_wrapper(path)
if query_string is None and path_s('?') in path:
path, query_string = path.split(path_s('?'), 1)
self.charset = charset
self.path = iri_to_uri(path)
if base_url is not None:
base_url = url_fix(iri_to_uri(base_url, charset), charset)
self.base_url = base_url
if isinstance(query_string, (bytes, text_type)):
self.query_string = query_string
else:
if query_string is None:
query_string = MultiDict()
elif not isinstance(query_string, MultiDict):
query_string = MultiDict(query_string)
self.args = query_string
self.method = method
if headers is None:
headers = Headers()
elif not isinstance(headers, Headers):
headers = Headers(headers)
self.headers = headers
if content_type is not None:
self.content_type = content_type
if errors_stream is None:
errors_stream = sys.stderr
self.errors_stream = errors_stream
self.multithread = multithread
self.multiprocess = multiprocess
self.run_once = run_once
self.environ_base = environ_base
self.environ_overrides = environ_overrides
self.input_stream = input_stream
self.content_length = content_length
self.closed = False
if data:
if input_stream is not None:
raise TypeError('can\'t provide input stream and data')
if isinstance(data, text_type):
data = data.encode(self.charset)
if isinstance(data, bytes):
self.input_stream = BytesIO(data)
if self.content_length is None:
self.content_length = len(data)
else:
for key, value in _iter_data(data):
if isinstance(value, (tuple, dict)) or \
hasattr(value, 'read'):
self._add_file_from_data(key, value)
else:
self.form.setlistdefault(key).append(value)
def __init__(self, args=None, view_args=None, endpoint=None):
self.args = MultiDict(args) if args else MultiDict()
self.view_args = view_args or {}
self.url_rule = FakeUrlRule(endpoint)
def test_libraries_post_create(self):
class TestFileUpload(StringIO):
headers = {"Content-Type": "image/png"}
image_data = '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x01\x03\x00\x00\x00%\xdbV\xca\x00\x00\x00\x06PLTE\xffM\x00\x01\x01\x01\x8e\x1e\xe5\x1b\x00\x00\x00\x01tRNS\xcc\xd24V\xfd\x00\x00\x00\nIDATx\x9cc`\x00\x00\x00\x02\x00\x01H\xaf\xa4q\x00\x00\x00\x00IEND\xaeB`\x82'
original_geographic_validate = GeographicValidator(
).validate_geographic_areas
class MockGeographicValidator(GeographicValidator):
def __init__(self):
self.was_called = False
def validate_geographic_areas(self, values, db):
self.was_called = True
return original_geographic_validate(values, db)
original_announcement_validate = AnnouncementListValidator(
).validate_announcements
class MockAnnouncementListValidator(AnnouncementListValidator):
def __init__(self):
self.was_called = False
def validate_announcements(self, values):
self.was_called = True
return original_announcement_validate(values)
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("name", "The New York Public Library"),
("short_name", "nypl"),
("library_description", "Short description of library"),
(Configuration.WEBSITE_URL, "https://library.library/"),
(Configuration.TINY_COLLECTION_LANGUAGES, ['ger']),
(Configuration.LIBRARY_SERVICE_AREA,
['06759', 'everywhere', 'MD', 'Boston, MA']),
(Configuration.LIBRARY_FOCUS_AREA,
['Manitoba', 'Broward County, FL', 'QC']),
(Announcements.SETTING_NAME,
json.dumps([self.active, self.forthcoming])),
(Configuration.DEFAULT_NOTIFICATION_EMAIL_ADDRESS,
"*****@*****.**"),
(Configuration.HELP_EMAIL, "*****@*****.**"),
(Configuration.FEATURED_LANE_SIZE, "5"),
(Configuration.DEFAULT_FACET_KEY_PREFIX +
FacetConstants.ORDER_FACET_GROUP_NAME,
FacetConstants.ORDER_RANDOM),
(Configuration.ENABLED_FACETS_KEY_PREFIX +
FacetConstants.ORDER_FACET_GROUP_NAME + "_" +
FacetConstants.ORDER_TITLE, ''),
(Configuration.ENABLED_FACETS_KEY_PREFIX +
FacetConstants.ORDER_FACET_GROUP_NAME + "_" +
FacetConstants.ORDER_RANDOM, ''),
])
flask.request.files = MultiDict([
(Configuration.LOGO, TestFileUpload(image_data)),
])
geographic_validator = MockGeographicValidator()
announcement_validator = MockAnnouncementListValidator()
validators = dict(
geographic=geographic_validator,
announcements=announcement_validator,
)
response = self.manager.admin_library_settings_controller.process_post(
validators)
eq_(response.status_code, 201)
library = get_one(self._db, Library, short_name="nypl")
eq_(library.uuid, response.response[0])
eq_(library.name, "The New York Public Library")
eq_(library.short_name, "nypl")
eq_(
"5",
ConfigurationSetting.for_library(Configuration.FEATURED_LANE_SIZE,
library).value)
eq_(
FacetConstants.ORDER_RANDOM,
ConfigurationSetting.for_library(
Configuration.DEFAULT_FACET_KEY_PREFIX +
FacetConstants.ORDER_FACET_GROUP_NAME, library).value)
eq_(
json.dumps(
[FacetConstants.ORDER_TITLE, FacetConstants.ORDER_RANDOM]),
ConfigurationSetting.for_library(
Configuration.ENABLED_FACETS_KEY_PREFIX +
FacetConstants.ORDER_FACET_GROUP_NAME, library).value)
eq_(
"data:image/png;base64,%s" % base64.b64encode(image_data),
ConfigurationSetting.for_library(Configuration.LOGO,
library).value)
eq_(geographic_validator.was_called, True)
eq_(
'{"CA": [], "US": ["06759", "everywhere", "MD", "Boston, MA"]}',
ConfigurationSetting.for_library(
Configuration.LIBRARY_SERVICE_AREA, library).value)
eq_(
'{"CA": ["Manitoba", "Quebec"], "US": ["Broward County, FL"]}',
ConfigurationSetting.for_library(Configuration.LIBRARY_FOCUS_AREA,
library).value)
# Announcements were validated.
eq_(announcement_validator.was_called, True)
# The validated result was written to the database, such that we can
# parse it as a list of Announcement objects.
announcements = Announcements.for_library(library).announcements
eq_([self.active['id'], self.forthcoming['id']],
[x.id for x in announcements])
assert all(isinstance(x, Announcement) for x in announcements)
# When the library was created, default lanes were also created
# according to its language setup. This library has one tiny
# collection (not a good choice for a real library), so only
# two lanes were created: "Other Languages" and then "German"
# underneath it.
[german, other_languages] = sorted(library.lanes,
key=lambda x: x.display_name)
eq_(None, other_languages.parent)
eq_(['ger'], other_languages.languages)
eq_(other_languages, german.parent)
eq_(['ger'], german.languages)
def test_patron_auth_services_post_edit(self):
mock_controller = self._get_mock()
l1, ignore = create(
self._db,
Library,
name="Library 1",
short_name="L1",
)
l2, ignore = create(
self._db,
Library,
name="Library 2",
short_name="L2",
)
auth_service, ignore = create(
self._db,
ExternalIntegration,
protocol=SimpleAuthenticationProvider.__module__,
goal=ExternalIntegration.PATRON_AUTH_GOAL,
)
auth_service.setting(
BasicAuthenticationProvider.TEST_IDENTIFIER).value = "old_user"
auth_service.setting(
BasicAuthenticationProvider.TEST_PASSWORD).value = "old_password"
auth_service.libraries = [l1]
with self.request_context_with_admin("/", method="POST"):
flask.request.form = MultiDict([
("id", auth_service.id),
("protocol", SimpleAuthenticationProvider.__module__),
(
"libraries",
json.dumps([{
"short_name":
l2.short_name,
AuthenticationProvider.EXTERNAL_TYPE_REGULAR_EXPRESSION:
"^(.)",
AuthenticationProvider.LIBRARY_IDENTIFIER_RESTRICTION_TYPE:
AuthenticationProvider.
LIBRARY_IDENTIFIER_RESTRICTION_TYPE_NONE,
AuthenticationProvider.LIBRARY_IDENTIFIER_FIELD:
AuthenticationProvider.
LIBRARY_IDENTIFIER_RESTRICTION_BARCODE,
}]),
),
] + self._common_basic_auth_arguments())
response = (self.manager.admin_patron_auth_services_controller.
process_patron_auth_services())
assert response.status_code == 200
assert mock_controller.validate_formats_call_count == 1
assert auth_service.id == int(response.response[0])
assert SimpleAuthenticationProvider.__module__ == auth_service.protocol
assert ("user" == auth_service.setting(
BasicAuthenticationProvider.TEST_IDENTIFIER).value)
assert ("pass" == auth_service.setting(
BasicAuthenticationProvider.TEST_PASSWORD).value)
assert [l2] == auth_service.libraries
assert (
"^(.)" == ConfigurationSetting.for_library_and_externalintegration(
self._db,
AuthenticationProvider.EXTERNAL_TYPE_REGULAR_EXPRESSION,
l2,
auth_service,
).value)