def test_publish_with_multiple_urls(self, mock):
data = MultiDict({'hub.mode': 'publish'})
data.add('hub.url', 'http://www.example.com/')
data.add('hub.url', 'http://www.site.com/')
request = self.r('/publish', self.valid_headers, POST=data)
info = publish(None, request)
self.assertEqual(info.status_code, 204)
def parse_path(self):
# parse path and parameters
path = self.path.split('?')[0]
params = self.path[len(path) + 1:].split('&')
param_dict = MultiDict()
for param in params:
key = param.split('=')[0]
value = param[len(key) + 1:]
param_dict.add(key, value)
return path, param_dict
def parse_path(self):
# parse path and parameters
path = self.path.split('?')[0]
params = self.path[len(path)+1:].split('&')
param_dict = MultiDict()
for param in params:
key = param.split('=')[0]
value = param[len(key)+1:]
param_dict.add(key, value)
return path, param_dict
def test_publish_fetches_topic_content(self, mock):
data = MultiDict({'hub.mode': 'publish'})
data.add('hub.url', 'http://www.example.com/')
data.add('hub.url', 'http://www.site.com/')
request = self.r('/publish', self.valid_headers, POST=data)
hub = request.root
info = publish(None, request)
first = hub.topics.get('http://www.example.com/')
second = hub.topics.get('http://www.site.com/')
self.assertEqual(info.status_code, 204)
self.assertTrue(first.timestamp is not None)
self.assertTrue(second.timestamp is not None)
def test_dict():
d = MultiDict({'a': 1})
assert list(d.items()) == [('a', 1)]
d['b'] = 2
d['c'] = 3
assert list(d.items()) == [('a', 1), ('b', 2), ('c', 3)]
d['b'] = 4
assert list(d.items()) == [('a', 1), ('c', 3), ('b', 4)]
d.add('b', 5)
assert_raises(KeyError, d.getone, "b")
assert d.getall('b') == [4, 5]
assert list(d.items()) == [('a', 1), ('c', 3), ('b', 4), ('b', 5)]
del d['b']
assert list(d.items()) == [('a', 1), ('c', 3)]
assert d.pop('xxx', 5) == 5
assert d.getone('a') == 1
assert d.popitem() == ('c', 3)
assert list(d.items()) == [('a', 1)]
item = []
assert d.setdefault('z', item) is item
assert list(d.items()) == [('a', 1), ('z', item)]
assert d.setdefault('y', 6) == 6
assert d.mixed() == {'a': 1, 'y': 6, 'z': item}
assert d.dict_of_lists() == {'a': [1], 'y': [6], 'z': [item]}
assert 'a' in d
dcopy = d.copy()
assert dcopy is not d
assert dcopy == d
d['x'] = 'x test'
assert dcopy != d
d[(1, None)] = (None, 1)
assert list(d.items()) == [('a', 1), ('z', []), ('y', 6), ('x', 'x test'),
((1, None), (None, 1))]
def copy(self):
""" Creates a copy of this ConfigStore. All configuration data is copied
over except for SQL connections.
"""
config_store = ConfigStore()
# Grab all ConfigDicts - even if they're actually ZATO_NONE - and make their copies
for attr_name in dir(self):
attr = getattr(self, attr_name)
if isinstance(attr, ConfigDict):
copy_func = getattr(attr, 'copy')
setattr(config_store, attr_name, copy_func())
elif attr is ZATO_NONE:
setattr(config_store, attr_name, ZATO_NONE)
http_soap = MultiDict()
dict_of_lists = self.http_soap.dict_of_lists()
for url_path, lists in dict_of_lists.items():
_info = Bunch()
for elem in lists:
for soap_action, item in elem.items():
_info[soap_action] = Bunch()
_info[soap_action].id = item.id
_info[soap_action].name = item.name
_info[soap_action].is_active = item.is_active
_info[soap_action].is_internal = item.is_internal
_info[soap_action].url_path = item.url_path
_info[soap_action].method = item.method
_info[soap_action].soap_version = item.soap_version
_info[soap_action].service_id = item.service_id
_info[soap_action].service_name = item.service_name
_info[soap_action].impl_name = item.impl_name
_info[soap_action].transport = item.transport
_info[soap_action].connection = item.connection
http_soap.add(url_path, _info)
config_store.http_soap = http_soap
config_store.url_sec = self.url_sec
config_store.broker_config = self.broker_config
config_store.odb_data = deepcopy(self.odb_data)
return config_store
def copy(self):
""" Creates a copy of this ConfigStore. All configuration data is copied
over except for SQL connections.
"""
config_store = ConfigStore()
# Grab all ConfigDicts - even if they're actually ZATO_NONE - and make their copies
for attr_name in dir(self):
attr = getattr(self, attr_name)
if isinstance(attr, ConfigDict):
copy_func = getattr(attr, 'copy')
setattr(config_store, attr_name, copy_func())
elif attr is ZATO_NONE:
setattr(config_store, attr_name, ZATO_NONE)
http_soap = MultiDict()
dict_of_lists = self.http_soap.dict_of_lists()
for url_path, lists in dict_of_lists.items():
_info = Bunch()
for elem in lists:
for soap_action, item in elem.items():
_info[soap_action] = Bunch()
_info[soap_action].id = item.id
_info[soap_action].name = item.name
_info[soap_action].is_active = item.is_active
_info[soap_action].is_internal = item.is_internal
_info[soap_action].url_path = item.url_path
_info[soap_action].method = item.method
_info[soap_action].soap_version = item.soap_version
_info[soap_action].service_id = item.service_id
_info[soap_action].service_name = item.service_name
_info[soap_action].impl_name = item.impl_name
_info[soap_action].transport = item.transport
_info[soap_action].connection = item.connection
http_soap.add(url_path, _info)
config_store.http_soap = http_soap
config_store.url_sec = self.url_sec
config_store.broker_config = self.broker_config
config_store.odb_data = deepcopy(self.odb_data)
return config_store
def test_dict():
d = MultiDict({'a': 1})
assert d.items() == [('a', 1)]
d['b'] = 2
d['c'] = 3
assert d.items() == [('a', 1), ('b', 2), ('c', 3)]
d['b'] = 4
assert d.items() == [('a', 1), ('c', 3), ('b', 4)]
d.add('b', 5)
raises(KeyError, 'd.getone("b")')
assert d.getall('b') == [4, 5]
assert d.items() == [('a', 1), ('c', 3), ('b', 4), ('b', 5)]
del d['b']
assert d.items() == [('a', 1), ('c', 3)]
assert d.pop('xxx', 5) == 5
assert d.getone('a') == 1
assert d.popitem() == ('c', 3)
assert d.items() == [('a', 1)]
item = []
assert d.setdefault('z', item) is item
assert d.items() == [('a', 1), ('z', item)]
assert d.setdefault('y', 6) == 6
assert d.mixed() == {'a': 1, 'y': 6, 'z': item}
assert d.dict_of_lists() == {'a': [1], 'y': [6], 'z': [item]}
assert 'a' in d
dcopy = d.copy()
assert dcopy is not d
assert dcopy == d
d['x'] = 'x test'
assert dcopy != d
d[(1, None)] = (None, 1)
assert d.items() == [('a', 1), ('z', []), ('y', 6), ('x', 'x test'),
((1, None), (None, 1))]
def _get_params(self, request):
'''
Returns the parameter from request, regardless their type
(GET OR POST) and returns a MultiDict containg the values.
'''
params = MultiDict()
if request.method == 'POST':
# this is required by aiohttp. without this,
# request.POST is empty
_ = [f for f in request.post()]
for k, v in request.POST.items():
# when data is transfered via POST, array names are
# appended a '[]'.
params.add(k.replace('[]', ''), v)
elif request.method == 'GET':
for k, v in request.GET.items():
params.add(k, v)
else:
raise TypeError('You need to pass a Request obj')
return params
class MimeHeaders(object):
"""Dictionary-like object that preserves the order and
supports multiple values for the same key, knows
whether it has been changed after the creation
"""
def __init__(self, items=()):
self.v = MultiDict([(normalize(key), val) for (key, val) in items])
self.changed = False
def __getitem__(self, key):
return self.v.get(normalize(key), None)
def __len__(self):
return len(self.v)
def __iter__(self):
return iter(self.v)
def __contains__(self, key):
return normalize(key) in self.v
def __setitem__(self, key, value):
self.v[normalize(key)] = _remove_newlines(value)
self.changed = True
def __delitem__(self, key):
del self.v[normalize(key)]
self.changed = True
def __nonzero__(self):
return len(self.v) > 0
def prepend(self, key, val):
self.v._items.insert(0, (key, _remove_newlines(val)))
self.changed = True
def add(self, key, value):
"""Adds header without changing the
existing headers with same name"""
self.v.add(normalize(key), _remove_newlines(value))
self.changed = True
def keys(self):
"""
Returns the keys. (message header names)
It remembers the order in which they were added, what
is really important
"""
return self.v.keys()
def transform(self, fn):
"""Accepts a function, getting a key, val and returning
a new pair of key, val and applies the function to all
header, value pairs in the message.
"""
changed = [False]
def tracking_fn(key, val):
new_key, new_val = fn(key, val)
if new_val != val or new_key != key:
changed[0] = True
return new_key, new_val
v = MultiDict(tracking_fn(key, val) for key, val in self.v.iteritems())
if changed[0]:
self.v = v
self.changed = True
def items(self):
"""
Returns header,val pairs in the preserved order.
"""
return list(self.iteritems())
def iteritems(self):
"""
Returns iterator header,val pairs in the preserved order.
"""
return self.v.iteritems()
def get(self, key, default=None):
"""
Returns header value (case-insensitive).
"""
return self.v.get(normalize(key), default)
def getall(self, key):
"""
Returns all header values by the given header name
(case-insensitive)
"""
return self.v.getall(normalize(key))
def have_changed(self):
"""Tells whether someone has altered the headers
after creation"""
return self.changed
def __str__(self):
return str(self.v)
@classmethod
def from_stream(cls, stream):
"""Takes a stream and reads the headers,
decodes headers to unicode dict like object"""
return cls(parse_stream(stream))
def to_stream(self, stream):
"""Takes a stream and serializes headers
in a mime format"""
for h, v in self.v.iteritems():
try:
h = h.encode('ascii')
except UnicodeDecodeError:
raise EncodingError("Non-ascii header name")
stream.write("{0}: {1}\r\n".format(h, to_mime(h, v)))
def parse_formvars(environ, include_get_vars=True):
"""Parses the request, returning a MultiDict of form variables.
If ``include_get_vars`` is true then GET (query string) variables
will also be folded into the MultiDict.
All values should be strings, except for file uploads which are
left as ``FieldStorage`` instances.
If the request was not a normal form request (e.g., a POST with an
XML body) then ``environ['wsgi.input']`` won't be read.
"""
source = environ['wsgi.input']
if 'paste.parsed_formvars' in environ:
parsed, check_source = environ['paste.parsed_formvars']
if check_source == source:
if include_get_vars:
parsed.update(parse_querystring(environ))
return parsed
# @@: Shouldn't bother FieldStorage parsing during GET/HEAD and
# fake_out_cgi requests
type = environ.get('CONTENT_TYPE', '').lower()
if ';' in type:
type = type.split(';', 1)[0]
fake_out_cgi = type not in ('', 'application/x-www-form-urlencoded',
'multipart/form-data')
# FieldStorage assumes a default CONTENT_LENGTH of -1, but a
# default of 0 is better:
if not environ.get('CONTENT_LENGTH'):
environ['CONTENT_LENGTH'] = '0'
# Prevent FieldStorage from parsing QUERY_STRING during GET/HEAD
# requests
old_query_string = environ.get('QUERY_STRING','')
environ['QUERY_STRING'] = ''
if fake_out_cgi:
input = StringIO('')
old_content_type = environ.get('CONTENT_TYPE')
old_content_length = environ.get('CONTENT_LENGTH')
environ['CONTENT_LENGTH'] = '0'
environ['CONTENT_TYPE'] = ''
else:
input = environ['wsgi.input']
fs = cgi.FieldStorage(fp=input,
environ=environ,
keep_blank_values=1)
environ['QUERY_STRING'] = old_query_string
if fake_out_cgi:
environ['CONTENT_TYPE'] = old_content_type
environ['CONTENT_LENGTH'] = old_content_length
formvars = MultiDict()
if isinstance(fs.value, list):
for name in fs.keys():
values = fs[name]
if not isinstance(values, list):
values = [values]
for value in values:
if not value.filename:
value = value.value
formvars.add(name, value)
environ['paste.parsed_formvars'] = (formvars, source)
if include_get_vars:
formvars.update(parse_querystring(environ))
return formvars
def search(self, ver=None, register=None):
log.debug('search %s params: %r', register, request.params)
if register == 'revision':
since_time = None
if 'since_id' in request.params:
id = request.params['since_id']
if not id:
return self._finish_bad_request(
_(u'No revision specified'))
rev = model.Session.query(model.Revision).get(id)
if rev is None:
return self._finish_not_found(
_(u'There is no revision with id: %s') % id)
since_time = rev.timestamp
elif 'since_time' in request.params:
since_time_str = request.params['since_time']
try:
since_time = h.date_str_to_datetime(since_time_str)
except ValueError as inst:
return self._finish_bad_request('ValueError: %s' % inst)
else:
return self._finish_bad_request(
_("Missing search term ('since_id=UUID' or " +
" 'since_time=TIMESTAMP')"))
revs = model.Session.query(model.Revision) \
.filter(model.Revision.timestamp > since_time) \
.order_by(model.Revision.timestamp) \
.limit(50) # reasonable enough for a page
return self._finish_ok([rev.id for rev in revs])
elif register in ['dataset', 'package', 'resource']:
try:
params = MultiDict(self._get_search_params(request.params))
except ValueError as e:
return self._finish_bad_request(
_('Could not read parameters: %r' % e))
# if using API v2, default to returning the package ID if
# no field list is specified
if register in ['dataset', 'package'] and not params.get('fl'):
params['fl'] = 'id' if ver == 2 else 'name'
try:
if register == 'resource':
query = search.query_for(model.Resource)
# resource search still uses ckan query parser
options = search.QueryOptions()
for k, v in params.items():
if (k in search.DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in search.DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
if isinstance(value, list):
values = value
for v in values:
query_fields.add(field, v)
results = query.run(
query=params.get('q'),
fields=query_fields,
options=options
)
else:
# For package searches in API v3 and higher, we can pass
# parameters straight to Solr.
if ver in [1, 2]:
# Otherwise, put all unrecognised ones into the q
# parameter
params = search.\
convert_legacy_parameters_to_solr(params)
query = search.query_for(model.Package)
# Remove any existing fq param and set the capacity to
# public
if 'fq' in params:
del params['fq']
params['fq'] = '+capacity:public'
# if callback is specified we do not want to send that to
# the search
if 'callback' in params:
del params['callback']
results = query.run(params)
return self._finish_ok(results)
except search.SearchError as e:
log.exception(e)
return self._finish_bad_request(
_('Bad search option: %s') % e)
else:
return self._finish_not_found(
_('Unknown register: %s') % register)
def parse_formvars(environ, include_get_vars=True):
"""Parses the request, returning a MultiDict of form variables.
If ``include_get_vars`` is true then GET (query string) variables
will also be folded into the MultiDict.
All values should be strings, except for file uploads which are
left as ``FieldStorage`` instances.
If the request was not a normal form request (e.g., a POST with an
XML body) then ``environ['wsgi.input']`` won't be read.
"""
source = environ['wsgi.input']
if 'paste.parsed_formvars' in environ:
parsed, check_source = environ['paste.parsed_formvars']
if check_source == source:
if include_get_vars:
parsed.update(parse_querystring(environ))
return parsed
# @@: Shouldn't bother FieldStorage parsing during GET/HEAD and
# fake_out_cgi requests
type = environ.get('CONTENT_TYPE', '').lower()
if ';' in type:
type = type.split(';', 1)[0]
fake_out_cgi = type not in ('', 'application/x-www-form-urlencoded',
'multipart/form-data')
# FieldStorage assumes a default CONTENT_LENGTH of -1, but a
# default of 0 is better:
if not environ.get('CONTENT_LENGTH'):
environ['CONTENT_LENGTH'] = '0'
# Prevent FieldStorage from parsing QUERY_STRING during GET/HEAD
# requests
old_query_string = environ.get('QUERY_STRING','')
environ['QUERY_STRING'] = ''
if fake_out_cgi:
input = six.BytesIO(b'')
old_content_type = environ.get('CONTENT_TYPE')
old_content_length = environ.get('CONTENT_LENGTH')
environ['CONTENT_LENGTH'] = '0'
environ['CONTENT_TYPE'] = ''
else:
input = environ['wsgi.input']
fs = cgi.FieldStorage(fp=input,
environ=environ,
keep_blank_values=1)
environ['QUERY_STRING'] = old_query_string
if fake_out_cgi:
environ['CONTENT_TYPE'] = old_content_type
environ['CONTENT_LENGTH'] = old_content_length
formvars = MultiDict()
if isinstance(fs.value, list):
for name in fs.keys():
values = fs[name]
if not isinstance(values, list):
values = [values]
for value in values:
if not value.filename:
value = value.value
formvars.add(name, value)
environ['paste.parsed_formvars'] = (formvars, source)
if include_get_vars:
formvars.update(parse_querystring(environ))
return formvars
class QueryParser(object):
"""
The query parser will take any incoming query specifications and turn
them into field-specific and general query parts.
"""
def __init__(self, query, terms, fields):
self._query = query
self._terms = terms
self._fields = MultiDict(fields)
@property
def query(self):
if not hasattr(self, '_combined_query'):
parts = [self._query if self._query is not None else '']
for term in self._terms:
if term.find(u' ') != -1:
term = u"\"%s\"" % term
parts.append(term.strip())
for field, value in self._fields.items():
if value.find(' ') != -1:
value = u"\"%s\"" % value
parts.append(u"%s:%s" % (field.strip(), value.strip()))
self._combined_query = u' '.join(parts)
return self._combined_query
def _query_tokens(self):
""" Split the query string, leaving quoted strings intact. """
if self._query:
inside_quote = False
buf = u''
for ch in self._query:
if ch == u' ' and not inside_quote:
if len(buf):
yield buf.strip()
buf = u''
elif ch == inside_quote:
inside_quote = False
elif ch in [u"\"", u"'"]:
inside_quote = ch
else:
buf += ch
if len(buf):
yield buf.strip()
def _parse_query(self):
""" Decompose the query string into fields and terms. """
self._combined_fields = MultiDict(self._fields)
self._combined_terms = list(self._terms)
for token in self._query_tokens():
colon_pos = token.find(u':')
if colon_pos != -1:
field = token[:colon_pos]
value = token[colon_pos+1:]
value = value.strip('"').strip("'").strip()
self._combined_fields.add(field, value)
else:
self._combined_terms.append(token)
@property
def fields(self):
if not hasattr(self, '_combined_fields'):
self._parse_query()
return self._combined_fields
@property
def terms(self):
if not hasattr(self, '_combined_terms'):
self._parse_query()
return self._combined_terms
def validate(self):
""" Check that this is a valid query. """
if not len(self.query):
raise SearchError("No query has been specified")
def __str__(self):
return self.query
def __repr__(self):
return "Query(%s)" % self
class MimeHeaders(object):
"""Dictionary-like object that preserves the order and
supports multiple values for the same key, knows
whether it has been changed after the creation
"""
def __init__(self, items=()):
self._v = MultiDict([(normalize(key), remove_newlines(val))
for (key, val) in items])
self.changed = False
def __getitem__(self, key):
return self._v.get(normalize(key), None)
def __len__(self):
return len(self._v)
def __iter__(self):
return iter(self._v)
def __contains__(self, key):
return normalize(key) in self._v
def __setitem__(self, key, value):
self._v[normalize(key)] = remove_newlines(value)
self.changed = True
def __delitem__(self, key):
del self._v[normalize(key)]
self.changed = True
def __nonzero__(self):
return len(self._v) > 0
def prepend(self, key, value):
self._v._items.insert(0, (normalize(key), remove_newlines(value)))
self.changed = True
def add(self, key, value):
"""Adds header without changing the
existing headers with same name"""
self._v.add(normalize(key), remove_newlines(value))
self.changed = True
def keys(self):
"""
Returns the keys. (message header names)
It remembers the order in which they were added, what
is really important
"""
return self._v.keys()
def transform(self, fn):
"""Accepts a function, getting a key, val and returning
a new pair of key, val and applies the function to all
header, value pairs in the message.
"""
changed = [False]
def tracking_fn(key, val):
new_key, new_val = fn(key, val)
if new_val != val or new_key != key:
changed[0] = True
return new_key, new_val
v = MultiDict(tracking_fn(key, val) for key, val in self._v.iteritems())
if changed[0]:
self._v = v
self.changed = True
def items(self):
"""
Returns header,val pairs in the preserved order.
"""
return list(self.iteritems())
def iteritems(self):
"""
Returns iterator header,val pairs in the preserved order.
"""
return self._v.iteritems()
def get(self, key, default=None):
"""
Returns header value (case-insensitive).
"""
return self._v.get(normalize(key), default)
def getall(self, key):
"""
Returns all header values by the given header name
(case-insensitive)
"""
return self._v.getall(normalize(key))
def have_changed(self):
"""Tells whether someone has altered the headers
after creation"""
return self.changed
def __str__(self):
return str(self._v)
@classmethod
def from_stream(cls, stream):
"""Takes a stream and reads the headers,
decodes headers to unicode dict like object"""
return cls(parse_stream(stream))
def to_stream(self, stream):
"""Takes a stream and serializes headers
in a mime format"""
for h, v in self._v.iteritems():
try:
h = h.encode('ascii')
except UnicodeDecodeError:
raise EncodingError("Non-ascii header name")
stream.write("{0}: {1}\r\n".format(h, to_mime(h, v)))
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in search.DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
if isinstance(value, list):
values = value
for v in values:
query_fields.add(field, v)
results = query.run(
query=params.get('q'),
fields=query_fields,
options=options
)
else:
# For package searches in API v3 and higher, we can pass
# parameters straight to Solr.
if ver in [1, 2]:
# Otherwise, put all unrecognised ones into the q
# parameter
params = search.\
convert_legacy_parameters_to_solr(params)
query = search.query_for(model.Package)
def get_url_security(self, cluster_id, connection=None):
""" Returns the security configuration of HTTP URLs.
"""
# What DB class to fetch depending on the string value of the security type.
sec_type_db_class = {
'tech_acc': TechnicalAccount,
'basic_auth': HTTPBasicAuth,
'wss': WSSDefinition
}
result = MultiDict()
query = http_soap_security_list(self._session, cluster_id, connection)
columns = Bunch()
# So ConfigDict has its data in the format it expects
for c in query.statement.columns:
columns[c.name] = None
for item in query.all():
_info = Bunch()
_info[item.soap_action] = Bunch()
_info[item.soap_action].is_active = item.is_active
_info[item.soap_action].transport = item.transport
_info[item.soap_action].data_format = item.data_format
if item.security_id:
_info[item.soap_action].sec_def = Bunch()
# Will raise KeyError if the DB gets somehow misconfigured.
db_class = sec_type_db_class[item.sec_type]
sec_def = self._session.query(db_class).\
filter(db_class.id==item.security_id).\
one()
# Common things first
_info[item.soap_action].sec_def.name = sec_def.name
_info[item.soap_action].sec_def.password = sec_def.password
_info[item.soap_action].sec_def.sec_type = item.sec_type
if item.sec_type == security_def_type.tech_account:
_info[item.soap_action].sec_def.salt = sec_def.salt
elif item.sec_type == security_def_type.basic_auth:
_info[item.soap_action].sec_def.username = sec_def.username
_info[item.soap_action].sec_def.password = sec_def.password
_info[item.soap_action].sec_def.realm = sec_def.realm
elif item.sec_type == security_def_type.wss:
_info[item.soap_action].sec_def.username = sec_def.username
_info[item.soap_action].sec_def.password = sec_def.password
_info[
item.
soap_action].sec_def.password_type = sec_def.password_type
_info[
item.
soap_action].sec_def.reject_empty_nonce_creat = sec_def.reject_empty_nonce_creat
_info[
item.
soap_action].sec_def.reject_stale_tokens = sec_def.reject_stale_tokens
_info[
item.
soap_action].sec_def.reject_expiry_limit = sec_def.reject_expiry_limit
_info[
item.
soap_action].sec_def.nonce_freshness_time = sec_def.nonce_freshness_time
else:
_info[item.soap_action].sec_def = ZATO_NONE
result.add(item.url_path, _info)
return result, columns
def search(self, ver=None, register=None):
log.debug('search %s params: %r', register, request.params)
if register == 'revision':
since_time = None
if 'since_id' in request.params:
id = request.params['since_id']
if not id:
return self._finish_bad_request(
_(u'No revision specified'))
rev = model.Session.query(model.Revision).get(id)
if rev is None:
return self._finish_not_found(
_(u'There is no revision with id: %s') % id)
since_time = rev.timestamp
elif 'since_time' in request.params:
since_time_str = request.params['since_time']
try:
since_time = h.date_str_to_datetime(since_time_str)
except ValueError as inst:
return self._finish_bad_request('ValueError: %s' % inst)
else:
return self._finish_bad_request(
_("Missing search term ('since_id=UUID' or " +
" 'since_time=TIMESTAMP')"))
revs = model.Session.query(model.Revision) \
.filter(model.Revision.timestamp > since_time) \
.order_by(model.Revision.timestamp) \
.limit(50) # reasonable enough for a page
return self._finish_ok([rev.id for rev in revs])
elif register in ['dataset', 'package', 'resource']:
try:
params = MultiDict(self._get_search_params(request.params))
except ValueError as e:
return self._finish_bad_request(
_('Could not read parameters: %r' % e))
# if using API v2, default to returning the package ID if
# no field list is specified
if register in ['dataset', 'package'] and not params.get('fl'):
params['fl'] = 'id' if ver == 2 else 'name'
try:
if register == 'resource':
query = search.query_for(model.Resource)
# resource search still uses ckan query parser
options = search.QueryOptions()
for k, v in params.items():
if (k in search.DEFAULT_OPTIONS.keys()):
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in search.DEFAULT_OPTIONS.keys() or \
field in IGNORE_FIELDS:
continue
values = [value]
if isinstance(value, list):
values = value
for v in values:
query_fields.add(field, v)
results = query.run(query=params.get('q'),
fields=query_fields,
options=options)
else:
# For package searches in API v3 and higher, we can pass
# parameters straight to Solr.
if ver in [1, 2]:
# Otherwise, put all unrecognised ones into the q
# parameter
params = search.\
convert_legacy_parameters_to_solr(params)
query = search.query_for(model.Package)
# Remove any existing fq param and set the capacity to
# public
if 'fq' in params:
del params['fq']
params['fq'] = '+capacity:public'
# if callback is specified we do not want to send that to
# the search
if 'callback' in params:
del params['callback']
results = query.run(params)
return self._finish_ok(results)
except search.SearchError as e:
log.exception(e)
return self._finish_bad_request(_('Bad search option: %s') % e)
else:
return self._finish_not_found(_('Unknown register: %s') % register)
def _after_init_accepted(self, server):
if self.singleton_server:
for(_, name, is_active, job_type, start_date, extra, service,\
_, weeks, days, hours, minutes, seconds, repeats, cron_definition)\
in self.odb.get_job_list(server.cluster.id):
if is_active:
job_data = Bunch({'name':name, 'is_active':is_active,
'job_type':job_type, 'start_date':start_date,
'extra':extra, 'service':service, 'weeks':weeks,
'days':days, 'hours':hours, 'minutes':minutes,
'seconds':seconds, 'repeats':repeats,
'cron_definition':cron_definition})
self.singleton_server.scheduler.create_edit('create', job_data)
# Start the connectors only once throughout the whole cluster
self._init_connectors(server)
# Mapping between SOAP actions and internal services.
#for soap_action, service_name in self.odb.get_internal_channel_list(server.cluster.id):
# self.request_handler.soap_handler.soap_config[soap_action] = service_name
# FTP
ftp_conn_params = Bunch()
for item in self.odb.get_out_ftp_list(server.cluster.id):
ftp_conn_params[item.name] = Bunch()
ftp_conn_params[item.name].is_active = item.is_active
ftp_conn_params[item.name].name = item.name
ftp_conn_params[item.name].host = item.host
ftp_conn_params[item.name].user = item.user
ftp_conn_params[item.name].password = item.password
ftp_conn_params[item.name].acct = item.acct
ftp_conn_params[item.name].timeout = item.timeout
ftp_conn_params[item.name].port = item.port
ftp_conn_params[item.name].dircache = item.dircache
self.ftp = FTPFacade(ftp_conn_params)
self.worker_config = Bunch()
# Repo location so that AMQP subprocesses know how where to read
# the server's configuration from.
self.worker_config.repo_location = self.repo_location
# The broker client for each of the worker threads.
self.worker_config.broker_config = Bunch()
self.worker_config.broker_config.name = 'worker-thread'
self.worker_config.broker_config.broker_token = self.broker_token
self.worker_config.broker_config.zmq_context = self.zmq_context
self.worker_config.broker_config.broker_push_client_pull = self.broker_push_worker_pull
self.worker_config.broker_config.client_push_broker_pull = self.worker_push_broker_pull
self.worker_config.broker_config.broker_pub_client_sub = self.broker_pub_worker_sub
# HTTP Basic Auth
ba_config = Bunch()
for item in self.odb.get_basic_auth_list(server.cluster.id):
ba_config[item.name] = Bunch()
ba_config[item.name].is_active = item.is_active
ba_config[item.name].username = item.username
ba_config[item.name].domain = item.domain
ba_config[item.name].password = item.password
# Technical accounts
ta_config = Bunch()
for item in self.odb.get_tech_acc_list(server.cluster.id):
ta_config[item.name] = Bunch()
ta_config[item.name].is_active = item.is_active
ta_config[item.name].name = item.name
ta_config[item.name].password = item.password
ta_config[item.name].salt = item.salt
wss_config = Bunch()
for item in self.odb.get_wss_list(server.cluster.id):
wss_config[item.name] = Bunch()
wss_config[item.name].is_active = item.is_active
wss_config[item.name].username = item.username
wss_config[item.name].password = item.password
wss_config[item.name].password_type = item.password_type
wss_config[item.name].reject_empty_nonce_ts = item.reject_empty_nonce_ts
wss_config[item.name].reject_stale_username = item.reject_stale_username
wss_config[item.name].expiry_limit = item.expiry_limit
wss_config[item.name].nonce_freshness = item.nonce_freshness
# Security configuration of HTTP URLs.
url_sec = self.odb.get_url_security(server)
# All the HTTP/SOAP channels.
http_soap = MultiDict()
for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'):
_info = Bunch()
_info[item.soap_action] = Bunch()
_info[item.soap_action].id = item.id
_info[item.soap_action].name = item.name
_info[item.soap_action].is_internal = item.is_internal
_info[item.soap_action].url_path = item.url_path
_info[item.soap_action].method = item.method
_info[item.soap_action].soap_version = item.soap_version
_info[item.soap_action].service_id = item.service_id
_info[item.soap_action].service_name = item.service_name
_info[item.soap_action].impl_name = item.impl_name
http_soap.add(item.url_path, _info)
self.worker_config.basic_auth = ba_config
self.worker_config.tech_acc = ta_config
self.worker_config.wss = wss_config
self.worker_config.url_sec = url_sec
self.worker_config.http_soap = http_soap
# The parallel server's broker client. The client's used to notify
# all the server's AMQP subprocesses that they need to shut down.
self.broker_client = BrokerClient()
self.broker_client.name = 'parallel'
self.broker_client.token = server.cluster.broker_token
self.broker_client.zmq_context = self.zmq_context
self.broker_client.client_push_broker_pull = self.parallel_push_broker_pull
self.broker_client.init()
self.broker_client.start()
class QueryParser(object):
"""
The query parser will take any incoming query specifications and turn
them into field-specific and general query parts.
"""
def __init__(self, query, terms, fields):
self._query = query
self._terms = terms
self._fields = MultiDict(fields)
@property
def query(self):
if not hasattr(self, '_combined_query'):
parts = [self._query if self._query is not None else '']
for term in self._terms:
if term.find(u' ') != -1:
term = u"\"%s\"" % term
parts.append(term.strip())
for field, value in self._fields.items():
if value.find(' ') != -1:
value = u"\"%s\"" % value
parts.append(u"%s:%s" % (field.strip(), value.strip()))
self._combined_query = u' '.join(parts)
return self._combined_query
def _query_tokens(self):
""" Split the query string, leaving quoted strings intact. """
if self._query:
inside_quote = False
buf = u''
for ch in self._query:
if ch == u' ' and not inside_quote:
if len(buf):
yield buf.strip()
buf = u''
elif ch == inside_quote:
inside_quote = False
elif ch in [u"\"", u"'"]:
inside_quote = ch
else:
buf += ch
if len(buf):
yield buf.strip()
def _parse_query(self):
""" Decompose the query string into fields and terms. """
self._combined_fields = MultiDict(self._fields)
self._combined_terms = list(self._terms)
for token in self._query_tokens():
colon_pos = token.find(u':')
if colon_pos != -1:
field = token[:colon_pos]
value = token[colon_pos + 1:]
value = value.strip('"').strip("'").strip()
self._combined_fields.add(field, value)
else:
self._combined_terms.append(token)
@property
def fields(self):
if not hasattr(self, '_combined_fields'):
self._parse_query()
return self._combined_fields
@property
def terms(self):
if not hasattr(self, '_combined_terms'):
self._parse_query()
return self._combined_terms
def validate(self):
""" Check that this is a valid query. """
pass
def __str__(self):
return self.query
def __repr__(self):
return "Query(%r)" % self.query
if k in search.DEFAULT_OPTIONS.keys():
options[k] = v
options.update(params)
options.username = c.user
options.search_tags = False
options.return_objects = False
query_fields = MultiDict()
for field, value in params.items():
field = field.strip()
if field in search.DEFAULT_OPTIONS.keys() or field in IGNORE_FIELDS:
continue
values = [value]
if isinstance(value, list):
values = value
for v in values:
query_fields.add(field, v)
results = query.run(query=params.get("q"), fields=query_fields, options=options)
else:
# For package searches in API v3 and higher, we can pass
# parameters straight to Solr.
if ver in [1, 2]:
# Otherwise, put all unrecognised ones into the q
# parameter
params = search.convert_legacy_parameters_to_solr(params)
query = search.query_for(model.Package)
# Remove any existing fq param and set the capacity to
# public
if "fq" in params:
del params["fq"]
mainkey = ''
d = MultiDict()
DEBUG = 0
for item in items:
cleanitem = item
thiskey = item.find('b') #, attrs={'img':'class bdtIcon'})
if thiskey:
value = str(item)
value = value.replace(str(thiskey), '')
value = cleaner(value)
if DEBUG:
print "X %s" % value
mainkey = thiskey
if value:
d.add(thiskey.text, value)
else:
if mainkey:
if DEBUG:
print mainkey.text
print item
d.add(mainkey.text, item)
# final result
shown = {}
xmlfile.write("<?xml version=\"1.0\"?>\n")
xmlfile.write("<entry xmlns=\"http://www.w3.org/2005/Atom\"\n")
xmlfile.write(" xmlns:dcterms=\"http://purl.org/dc/terms/\">\n")
xmlfile.write(" <dcterms:source>%s</dcterms:source>\n" % inturl)
def get_url_security(self, cluster_id):
""" Returns the security configuration of HTTP URLs.
"""
# What DB class to fetch depending on the string value of the security type.
sec_type_db_class = {
'tech_acc': TechnicalAccount,
'basic_auth': HTTPBasicAuth,
'wss': WSSDefinition
}
result = MultiDict()
query = http_soap_security_list(self._session, cluster_id)
columns = Bunch()
# So ConfigDict has its data in the format it expects
for c in query.statement.columns:
columns[c.name] = None
for item in query.all():
_info = Bunch()
_info[item.soap_action] = Bunch()
_info[item.soap_action].transport = item.transport
_info[item.soap_action].data_format = item.data_format
if item.security_id:
_info[item.soap_action].sec_def = Bunch()
# Will raise KeyError if the DB gets somehow misconfigured.
db_class = sec_type_db_class[item.sec_type]
sec_def = self._session.query(db_class).\
filter(db_class.id==item.security_id).\
one()
# Common things first
_info[item.soap_action].sec_def.name = sec_def.name
_info[item.soap_action].sec_def.password = sec_def.password
_info[item.soap_action].sec_def.sec_type = item.sec_type
if item.sec_type == security_def_type.tech_account:
_info[item.soap_action].sec_def.salt = sec_def.salt
elif item.sec_type == security_def_type.basic_auth:
_info[item.soap_action].sec_def.username = sec_def.username
_info[item.soap_action].sec_def.password = sec_def.password
_info[item.soap_action].sec_def.realm = sec_def.realm
elif item.sec_type == security_def_type.wss:
_info[item.soap_action].sec_def.username = sec_def.username
_info[item.soap_action].sec_def.password = sec_def.password
_info[item.soap_action].sec_def.password_type = sec_def.password_type
_info[item.soap_action].sec_def.reject_empty_nonce_creat = sec_def.reject_empty_nonce_creat
_info[item.soap_action].sec_def.reject_stale_tokens = sec_def.reject_stale_tokens
_info[item.soap_action].sec_def.reject_expiry_limit = sec_def.reject_expiry_limit
_info[item.soap_action].sec_def.nonce_freshness_time = sec_def.nonce_freshness_time
else:
_info[item.soap_action].sec_def = ZATO_NONE
result.add(item.url_path, _info)
return result, columns
