def test_iri_support():
"""The IRI support"""
assert_raises(UnicodeError, uri_to_iri, u'http://föö.com/')
assert_raises(UnicodeError, iri_to_uri, 'http://föö.com/')
assert uri_to_iri('http://xn--n3h.net/') == u'http://\u2603.net/'
assert uri_to_iri('http://%C3%BCser:p%C3%[email protected]/p%C3%A5th') == \
u'http://\xfcser:p\xe4ssword@\u2603.net/p\xe5th'
assert iri_to_uri(u'http://☃.net/') == 'http://xn--n3h.net/'
assert iri_to_uri(u'http://üser:pässword@☃.net/påth') == \
'http://%C3%BCser:p%C3%[email protected]/p%C3%A5th'
assert uri_to_iri('http://test.com/%3Fmeh?foo=%26%2F') == \
u'http://test.com/%3Fmeh?foo=%26%2F'
def _ensure_api():
access_token = flask.session.get("ig.access_token", None)
if access_token: return
api = _get_api()
url = flask.request.url
url = werkzeug.iri_to_uri(url)
return api.oauth_authorize(state=url)
def _ensure_api():
access_token = flask.session.get("ig.access_token", None)
if access_token: return
api = _get_api()
url = flask.request.url
url = werkzeug.iri_to_uri(url)
return api.oauth_authorize(state = url)
def test_iri_support():
"""The IRI support"""
assert_raises(UnicodeError, uri_to_iri, u'http://föö.com/')
assert_raises(UnicodeError, iri_to_uri, 'http://föö.com/')
assert uri_to_iri('http://xn--n3h.net/') == u'http://\u2603.net/'
assert uri_to_iri('http://%C3%BCser:p%C3%[email protected]/p%C3%A5th') == \
u'http://\xfcser:p\xe4ssword@\u2603.net/p\xe5th'
assert iri_to_uri(u'http://☃.net/') == 'http://xn--n3h.net/'
assert iri_to_uri(u'http://üser:pässword@☃.net/påth') == \
'http://%C3%BCser:p%C3%[email protected]/p%C3%A5th'
assert uri_to_iri('http://test.com/%3Fmeh?foo=%26%2F') == \
u'http://test.com/%3Fmeh?foo=%26%2F'
# this should work as well, might break on 2.4 because of a broken
# idna codec
assert uri_to_iri('/foo') == u'/foo'
assert iri_to_uri(u'/foo') == '/foo'
def suggest(self):
url = 'http://%s:%s/%s/' % (self.host, self.port, self.application)
if self.core != '':
url += '%s/' % self.core
params = '%s?spellcheck.q=%s&wt=%s&json.nl=%s&omitHeader=%s' % (
self.handler, urllib.urlencode(self.suggest_query), self.writer,
self.json_nl, self.omitHeader)
self.request_url = '%s%s' % (url, params)
# self.response = eval(urllib.request.urlopen(iri_to_uri(self.request_url)).read())
self.response = eval(requests.get(iri_to_uri(self.request_url)).text)
self.suggestions = self.response.get('spellcheck').get('suggestions')
def suggest(self):
url = 'http://%s:%s/%s/' % (self.host, self.port, self.application)
if self.core != '':
url += '%s/' % self.core
params = '%s?spellcheck.q=%s&wt=%s&json.nl=%s&omitHeader=%s' % (self.handler,
urllib.urlencode(
self.suggest_query),
self.writer, self.json_nl, self.omitHeader)
self.request_url = '%s%s' % (url, params)
# self.response = eval(urllib.request.urlopen(iri_to_uri(self.request_url)).read())
self.response = eval(requests.get(iri_to_uri(self.request_url)).text)
self.suggestions = self.response.get('spellcheck').get('suggestions')
def request(self):
params = ''
url = 'http://%s:%s/%s/' % (self.host, self.port, self.application)
if self.core != '':
url += '%s/' % self.core
fuzzy_tilde = ''
if self.fuzzy == 'true':
fuzzy_tilde = '~'
#logging.info(self.facet)
#logging.info(self.facet_fields)
if self.facet == 'true': # Old-style facetting...
facets = '&facet.field='.join(self.facet_fields)
params = '%s?q=%s%s&wt=%s&start=%s&rows=%s&facet.limit=%s&facet.mincount=%s&facet.offset=%s&facet.field=%s&json.nl=%s&facet=%s&facet.sort=%s&omitHeader=%s&defType=%s&facet.threads=-1' % (
self.handler, self.query, fuzzy_tilde, self.writer, self.start,
self.rows, self.facet_limit, self.facet_mincount,
self.facet_offset, facets, self.json_nl, self.facet,
self.facet_sort, self.omitHeader, self.defType)
if self.boost_most_recent == 'true':
params += '&boost=recip(ms(NOW/YEAR,year_boost),3.16e-11,1,1)'
# params = '%s/%s?q=%s%s&wt=%s&start=%s&rows=%s&facet.limit=%s&facet.mincount=%s&facet.offset=%s&facet.field=%s&json.nl=%s&facet=%s&facet.sort=%s&omitHeader=%s&defType=%s' % (
# self.application, self.handler, self.query, fuzzy_tilde, self.writer, self.start, self.rows, self.facet_limit,
# self.facet_mincount, self.facet_offset, facets, self.json_nl, self.facet, self.facet_sort,
# self.omitHeader, self.defType)
if len(self.sort_facet_by_index) > 0:
for sortfield in self.sort_facet_by_index:
params += '&f.%s.facet.sort=homepage&f.%s.facet.limit=-1' % (
sortfield, sortfield
) # Stupid hack until SOLR-1672 gets fixed
# Pivot needs a mincount of 0 for empty categories. Build mincounts of 1 for normal facets...
for myfacet in self.facet_fields:
params += '&f.%s.facet.mincount=1' % myfacet
if len(self.facet_tree) > 0:
params += '&facet.pivot='
for field in self.facet_tree:
# params += '&facet.pivot=%s,%s' % (self.facet_tree[0], self.facet_tree[1])
params += '%s,' % field
params = params[:-1]
else:
params = '%s?q=%s%s&wt=%s&start=%s&rows=%s&json.nl=%s&omitHeader=%s&defType=%s' % (
self.handler, self.query, fuzzy_tilde, self.writer, self.start,
self.rows, self.json_nl, self.omitHeader, self.defType)
if self.boost_most_recent == 'true':
params += '&boost=recip(ms(NOW/YEAR,year_boost),3.16e-11,1,1)'
if self.writer == 'csv':
params += '&csv.separator=%s' % self.csv_separator
# logging.info(self.json_facet)
if self.json_facet:
params += '&json.facet=%s' % (json.dumps(self.json_facet))
if len(self.fquery) > 0:
for fq in (self.fquery):
try:
# params += '&fq=%s' % urllib.parse.unquote(fq.encode('utf8'))
val = urllib.parse.unquote(fq).replace('#', '\%23')
# logging.info('%s >> %s' % (fq,val))
params += '&fq=%s' % val
except UnicodeDecodeError:
params += '&fq=%s' % urllib.parse.unquote(fq)
if self.sort:
# if self.cursor:
# params += '&sort=katkey+asc&cursorMark=%s' % self.cursor
# elif self.sort != 'score desc':
if self.sort != 'score desc':
params += '&sort=%s' % self.sort
if len(self.fields) > 0:
if self.application == 'elevate':
self.fields.append('[elevated]')
params += '&fl=%s' % '+'.join(self.fields)
if self.mlt is True:
self.facet = 'false'
mparams = '%s?q=%s&mlt=true&mlt.fl=%s&mlt.count=10&fl=%s&wt=%s&defType=%s' % (
self.handler, self.query, '+'.join(self.mlt_fields), '+'.join(
self.fields), self.writer, self.defType)
# if self.boost_most_recent == 'true':
# params += '&boost=recip(ms(NOW/YEAR,year_boost),3.16e-11,1,1)'
# self.response = eval(urllib.request.urlopen('%s%s' % (url, mparams)).read())
# logging.info(url)
# logging.info(mparams)
self.response = eval(requests.get('%s%s' % (url, mparams)).text)
for mlt in self.response.get('moreLikeThis'):
self.mlt_results = self.response.get('moreLikeThis').get(
mlt).get('docs')
if self.spellcheck == 'true':
params += '&spellcheck=true&spellcheck.collate=%s&spellcheck.count=%s' % (
self.spellcheck_collate, self.spellcheck_count)
if self.group[0]:
params += '&group=true&group.field=%s&group.limit=%s&group.sort=%s&group.ngroups=%s' % (
self.group_field[0], self.group_limit[0], self.group_sort[0],
self.group_ngroups[0])
if self.coordinates != '0,0':
try:
params += '&pt=%s&sfield=geolocation&fl=*+dist_:geodist()' % self.coordinates
except UnicodeDecodeError:
params += '&pt=%s&sfield=geolocation&fl=*+dist_:geodist()' % self.coordinates.decode(
'utf8')
if self.queryField:
params += '&qf=%s' % self.queryField
if self.stats == 'true':
params += '&stats=true&stats.field=' + '&stats.field='.join(
self.stats_fl)
if self.handler != 'query':
params += '&q.op=AND'
self.request_url = '%s%s' % (url, params)
# logging.fatal(iri_to_uri(self.request_url))
# logging.info('REQUEST: %s' % self.request_url)
if self.compress:
import urllib2
import StringIO
import gzip
request = urllib2.Request(iri_to_uri(self.request_url))
request.add_header('Accept-encoding', 'gzip')
opener = urllib2.build_opener()
compresseddata = opener.open(request).read()
compressedstream = StringIO.StringIO(compresseddata)
gzipper = gzip.GzipFile(fileobj=compressedstream)
self.response = eval(gzipper.read())
else:
# logging.error(self.request_url)
try:
# self.response = eval(urllib.request.urlopen(iri_to_uri(self.request_url)).read())
self.response = eval(
requests.get(iri_to_uri(self.request_url)).text)
except NameError:
# self.response = urllib.request.urlopen(iri_to_uri(self.request_url)).read()
self.response = requests.get(iri_to_uri(self.request_url)).text
except SyntaxError:
# self.response = urllib.request.urlopen(iri_to_uri(self.request_url)).read()
self.response = requests.get(iri_to_uri(self.request_url)).text
# self.response = eval(urllib.request.urlopen(self.request_url).read())
# logging.error(self.response)
try:
self.results = self.response.get('response').get('docs')
except AttributeError: # Grouped results...
# logging.fatal(e)
# logging.error(self.response)
try:
if self.response.get('grouped'):
self.results = self.response.get('grouped').get(
self.group_field[0]).get('groups')
except AttributeError:
pass
if self.facet == 'true':
self.facets = self.response.get('facet_counts').get('facet_fields')
# logging.info(self.facets)
if len(self.facet_tree) > 0:
self.tree = self.response.get('facet_counts').get('facet_pivot')
if self.json_facet:
# logging.info(self.response.get('facets'))
self.facets = self.response.get('facets')
if self.spellcheck == 'true' or self.handler.endswith('suggest'):
try:
self.suggestions = self.response.get('spellcheck').get(
'suggestions')
except AttributeError:
pass
if self.omitHeader != 'true':
self.qtime = float(
self.response.get('responseHeader').get('QTime')) / 1000
def bounce(key):
try:
uri = store[key]
return redirect(iri_to_uri(uri))
except KeyError as e:
return jsonify({"error": "url not found"}, 400)
def request(self):
params = ''
url = 'http://%s:%s/%s/' % (self.host, self.port, self.application)
if self.core != '':
url += '%s/' % self.core
fuzzy_tilde = ''
if self.fuzzy == 'true':
fuzzy_tilde = '~'
#logging.info(self.facet)
#logging.info(self.facet_fields)
if self.facet == 'true': # Old-style facetting...
facets = '&facet.field='.join(self.facet_fields)
params = '%s?q=%s%s&wt=%s&start=%s&rows=%s&facet.limit=%s&facet.mincount=%s&facet.offset=%s&facet.field=%s&json.nl=%s&facet=%s&facet.sort=%s&omitHeader=%s&defType=%s&facet.threads=-1' % (
self.handler, self.query, fuzzy_tilde, self.writer, self.start, self.rows, self.facet_limit,
self.facet_mincount, self.facet_offset, facets, self.json_nl, self.facet, self.facet_sort,
self.omitHeader, self.defType)
if self.boost_most_recent == 'true':
params += '&boost=recip(ms(NOW/YEAR,year_boost),3.16e-11,1,1)'
# params = '%s/%s?q=%s%s&wt=%s&start=%s&rows=%s&facet.limit=%s&facet.mincount=%s&facet.offset=%s&facet.field=%s&json.nl=%s&facet=%s&facet.sort=%s&omitHeader=%s&defType=%s' % (
# self.application, self.handler, self.query, fuzzy_tilde, self.writer, self.start, self.rows, self.facet_limit,
# self.facet_mincount, self.facet_offset, facets, self.json_nl, self.facet, self.facet_sort,
# self.omitHeader, self.defType)
if len(self.sort_facet_by_index) > 0:
for sortfield in self.sort_facet_by_index:
params += '&f.%s.facet.sort=homepage&f.%s.facet.limit=-1' % (
sortfield, sortfield)# Stupid hack until SOLR-1672 gets fixed
# Pivot needs a mincount of 0 for empty categories. Build mincounts of 1 for normal facets...
for myfacet in self.facet_fields:
params += '&f.%s.facet.mincount=1' % myfacet
if len(self.facet_tree) > 0:
params += '&facet.pivot='
for field in self.facet_tree:
# params += '&facet.pivot=%s,%s' % (self.facet_tree[0], self.facet_tree[1])
params += '%s,' % field
params = params[:-1]
else:
params = '%s?q=%s%s&wt=%s&start=%s&rows=%s&json.nl=%s&omitHeader=%s&defType=%s' % (
self.handler, self.query, fuzzy_tilde, self.writer, self.start, self.rows, self.json_nl,
self.omitHeader, self.defType)
if self.boost_most_recent == 'true':
params += '&boost=recip(ms(NOW/YEAR,year_boost),3.16e-11,1,1)'
if self.writer == 'csv':
params += '&csv.separator=%s' % self.csv_separator
# logging.info(self.json_facet)
if self.json_facet:
params += '&json.facet=%s' % (json.dumps(self.json_facet))
if len(self.fquery) > 0:
for fq in (self.fquery):
try:
# params += '&fq=%s' % urllib.parse.unquote(fq.encode('utf8'))
val = urllib.parse.unquote(fq).replace('#', '\%23')
# logging.info('%s >> %s' % (fq,val))
params += '&fq=%s' % val
except UnicodeDecodeError:
params += '&fq=%s' % urllib.parse.unquote(fq)
if self.sort:
# if self.cursor:
# params += '&sort=katkey+asc&cursorMark=%s' % self.cursor
# elif self.sort != 'score desc':
if self.sort != 'score desc':
params += '&sort=%s' % self.sort
if len(self.fields) > 0:
if self.application == 'elevate':
self.fields.append('[elevated]')
params += '&fl=%s' % '+'.join(self.fields)
if self.mlt is True:
self.facet = 'false'
mparams = '%s?q=%s&mlt=true&mlt.fl=%s&mlt.count=10&fl=%s&wt=%s&defType=%s' % (
self.handler, self.query, '+'.join(self.mlt_fields), '+'.join(self.fields),
self.writer, self.defType)
# if self.boost_most_recent == 'true':
# params += '&boost=recip(ms(NOW/YEAR,year_boost),3.16e-11,1,1)'
# self.response = eval(urllib.request.urlopen('%s%s' % (url, mparams)).read())
# logging.info(url)
# logging.info(mparams)
self.response = eval(requests.get('%s%s' % (url, mparams)).text)
for mlt in self.response.get('moreLikeThis'):
self.mlt_results = self.response.get('moreLikeThis').get(mlt).get('docs')
if self.spellcheck == 'true':
params += '&spellcheck=true&spellcheck.collate=%s&spellcheck.count=%s' % (
self.spellcheck_collate, self.spellcheck_count)
if self.group[0]:
params += '&group=true&group.field=%s&group.limit=%s&group.sort=%s&group.ngroups=%s' % (
self.group_field[0], self.group_limit[0], self.group_sort[0], self.group_ngroups[0])
if self.coordinates != '0,0':
try:
params += '&pt=%s&sfield=geolocation&fl=*+dist_:geodist()' % self.coordinates
except UnicodeDecodeError:
params += '&pt=%s&sfield=geolocation&fl=*+dist_:geodist()' % self.coordinates.decode('utf8')
if self.queryField:
params += '&qf=%s' % self.queryField
if self.stats == 'true':
params += '&stats=true&stats.field=' + '&stats.field='.join(self.stats_fl)
if self.handler != 'query':
params += '&q.op=AND'
self.request_url = '%s%s' % (url, params)
# logging.fatal(iri_to_uri(self.request_url))
# logging.info('REQUEST: %s' % self.request_url)
if self.compress:
import urllib2
import StringIO
import gzip
request = urllib2.Request(iri_to_uri(self.request_url))
request.add_header('Accept-encoding', 'gzip')
opener = urllib2.build_opener()
compresseddata = opener.open(request).read()
compressedstream = StringIO.StringIO(compresseddata)
gzipper = gzip.GzipFile(fileobj=compressedstream)
self.response = eval(gzipper.read())
else:
# logging.error(self.request_url)
try:
# self.response = eval(urllib.request.urlopen(iri_to_uri(self.request_url)).read())
self.response = eval(requests.get(iri_to_uri(self.request_url)).text)
except NameError:
# self.response = urllib.request.urlopen(iri_to_uri(self.request_url)).read()
self.response = requests.get(iri_to_uri(self.request_url)).text
except SyntaxError:
# self.response = urllib.request.urlopen(iri_to_uri(self.request_url)).read()
self.response = requests.get(iri_to_uri(self.request_url)).text
# self.response = eval(urllib.request.urlopen(self.request_url).read())
# logging.error(self.response)
try:
self.results = self.response.get('response').get('docs')
except AttributeError: # Grouped results...
# logging.fatal(e)
# logging.error(self.response)
try:
if self.response.get('grouped'):
self.results = self.response.get('grouped').get(self.group_field[0]).get('groups')
except AttributeError:
pass
if self.facet == 'true':
self.facets = self.response.get('facet_counts').get('facet_fields')
# logging.info(self.facets)
if len(self.facet_tree) > 0:
self.tree = self.response.get('facet_counts').get('facet_pivot')
if self.json_facet:
# logging.info(self.response.get('facets'))
self.facets = self.response.get('facets')
if self.spellcheck == 'true' or self.handler.endswith('suggest'):
try:
self.suggestions = self.response.get('spellcheck').get('suggestions')
except AttributeError:
pass
if self.omitHeader != 'true':
self.qtime = float(self.response.get('responseHeader').get('QTime')) / 1000