def test_publish_with_multiple_urls(self, mock): data = MultiDict({'hub.mode': 'publish'}) data.add('hub.url', 'http://www.example.com/') data.add('hub.url', 'http://www.site.com/') request = self.r('/publish', self.valid_headers, POST=data) info = publish(None, request) self.assertEqual(info.status_code, 204)
def parse_path(self): # parse path and parameters path = self.path.split('?')[0] params = self.path[len(path) + 1:].split('&') param_dict = MultiDict() for param in params: key = param.split('=')[0] value = param[len(key) + 1:] param_dict.add(key, value) return path, param_dict
def parse_path(self): # parse path and parameters path = self.path.split('?')[0] params = self.path[len(path)+1:].split('&') param_dict = MultiDict() for param in params: key = param.split('=')[0] value = param[len(key)+1:] param_dict.add(key, value) return path, param_dict
def test_publish_fetches_topic_content(self, mock): data = MultiDict({'hub.mode': 'publish'}) data.add('hub.url', 'http://www.example.com/') data.add('hub.url', 'http://www.site.com/') request = self.r('/publish', self.valid_headers, POST=data) hub = request.root info = publish(None, request) first = hub.topics.get('http://www.example.com/') second = hub.topics.get('http://www.site.com/') self.assertEqual(info.status_code, 204) self.assertTrue(first.timestamp is not None) self.assertTrue(second.timestamp is not None)
def test_dict(): d = MultiDict({'a': 1}) assert list(d.items()) == [('a', 1)] d['b'] = 2 d['c'] = 3 assert list(d.items()) == [('a', 1), ('b', 2), ('c', 3)] d['b'] = 4 assert list(d.items()) == [('a', 1), ('c', 3), ('b', 4)] d.add('b', 5) assert_raises(KeyError, d.getone, "b") assert d.getall('b') == [4, 5] assert list(d.items()) == [('a', 1), ('c', 3), ('b', 4), ('b', 5)] del d['b'] assert list(d.items()) == [('a', 1), ('c', 3)] assert d.pop('xxx', 5) == 5 assert d.getone('a') == 1 assert d.popitem() == ('c', 3) assert list(d.items()) == [('a', 1)] item = [] assert d.setdefault('z', item) is item assert list(d.items()) == [('a', 1), ('z', item)] assert d.setdefault('y', 6) == 6 assert d.mixed() == {'a': 1, 'y': 6, 'z': item} assert d.dict_of_lists() == {'a': [1], 'y': [6], 'z': [item]} assert 'a' in d dcopy = d.copy() assert dcopy is not d assert dcopy == d d['x'] = 'x test' assert dcopy != d d[(1, None)] = (None, 1) assert list(d.items()) == [('a', 1), ('z', []), ('y', 6), ('x', 'x test'), ((1, None), (None, 1))]
def copy(self): """ Creates a copy of this ConfigStore. All configuration data is copied over except for SQL connections. """ config_store = ConfigStore() # Grab all ConfigDicts - even if they're actually ZATO_NONE - and make their copies for attr_name in dir(self): attr = getattr(self, attr_name) if isinstance(attr, ConfigDict): copy_func = getattr(attr, 'copy') setattr(config_store, attr_name, copy_func()) elif attr is ZATO_NONE: setattr(config_store, attr_name, ZATO_NONE) http_soap = MultiDict() dict_of_lists = self.http_soap.dict_of_lists() for url_path, lists in dict_of_lists.items(): _info = Bunch() for elem in lists: for soap_action, item in elem.items(): _info[soap_action] = Bunch() _info[soap_action].id = item.id _info[soap_action].name = item.name _info[soap_action].is_active = item.is_active _info[soap_action].is_internal = item.is_internal _info[soap_action].url_path = item.url_path _info[soap_action].method = item.method _info[soap_action].soap_version = item.soap_version _info[soap_action].service_id = item.service_id _info[soap_action].service_name = item.service_name _info[soap_action].impl_name = item.impl_name _info[soap_action].transport = item.transport _info[soap_action].connection = item.connection http_soap.add(url_path, _info) config_store.http_soap = http_soap config_store.url_sec = self.url_sec config_store.broker_config = self.broker_config config_store.odb_data = deepcopy(self.odb_data) return config_store
def test_dict(): d = MultiDict({'a': 1}) assert d.items() == [('a', 1)] d['b'] = 2 d['c'] = 3 assert d.items() == [('a', 1), ('b', 2), ('c', 3)] d['b'] = 4 assert d.items() == [('a', 1), ('c', 3), ('b', 4)] d.add('b', 5) raises(KeyError, 'd.getone("b")') assert d.getall('b') == [4, 5] assert d.items() == [('a', 1), ('c', 3), ('b', 4), ('b', 5)] del d['b'] assert d.items() == [('a', 1), ('c', 3)] assert d.pop('xxx', 5) == 5 assert d.getone('a') == 1 assert d.popitem() == ('c', 3) assert d.items() == [('a', 1)] item = [] assert d.setdefault('z', item) is item assert d.items() == [('a', 1), ('z', item)] assert d.setdefault('y', 6) == 6 assert d.mixed() == {'a': 1, 'y': 6, 'z': item} assert d.dict_of_lists() == {'a': [1], 'y': [6], 'z': [item]} assert 'a' in d dcopy = d.copy() assert dcopy is not d assert dcopy == d d['x'] = 'x test' assert dcopy != d d[(1, None)] = (None, 1) assert d.items() == [('a', 1), ('z', []), ('y', 6), ('x', 'x test'), ((1, None), (None, 1))]
def _get_params(self, request): ''' Returns the parameter from request, regardless their type (GET OR POST) and returns a MultiDict containg the values. ''' params = MultiDict() if request.method == 'POST': # this is required by aiohttp. without this, # request.POST is empty _ = [f for f in request.post()] for k, v in request.POST.items(): # when data is transfered via POST, array names are # appended a '[]'. params.add(k.replace('[]', ''), v) elif request.method == 'GET': for k, v in request.GET.items(): params.add(k, v) else: raise TypeError('You need to pass a Request obj') return params
class MimeHeaders(object): """Dictionary-like object that preserves the order and supports multiple values for the same key, knows whether it has been changed after the creation """ def __init__(self, items=()): self.v = MultiDict([(normalize(key), val) for (key, val) in items]) self.changed = False def __getitem__(self, key): return self.v.get(normalize(key), None) def __len__(self): return len(self.v) def __iter__(self): return iter(self.v) def __contains__(self, key): return normalize(key) in self.v def __setitem__(self, key, value): self.v[normalize(key)] = _remove_newlines(value) self.changed = True def __delitem__(self, key): del self.v[normalize(key)] self.changed = True def __nonzero__(self): return len(self.v) > 0 def prepend(self, key, val): self.v._items.insert(0, (key, _remove_newlines(val))) self.changed = True def add(self, key, value): """Adds header without changing the existing headers with same name""" self.v.add(normalize(key), _remove_newlines(value)) self.changed = True def keys(self): """ Returns the keys. (message header names) It remembers the order in which they were added, what is really important """ return self.v.keys() def transform(self, fn): """Accepts a function, getting a key, val and returning a new pair of key, val and applies the function to all header, value pairs in the message. """ changed = [False] def tracking_fn(key, val): new_key, new_val = fn(key, val) if new_val != val or new_key != key: changed[0] = True return new_key, new_val v = MultiDict(tracking_fn(key, val) for key, val in self.v.iteritems()) if changed[0]: self.v = v self.changed = True def items(self): """ Returns header,val pairs in the preserved order. """ return list(self.iteritems()) def iteritems(self): """ Returns iterator header,val pairs in the preserved order. """ return self.v.iteritems() def get(self, key, default=None): """ Returns header value (case-insensitive). """ return self.v.get(normalize(key), default) def getall(self, key): """ Returns all header values by the given header name (case-insensitive) """ return self.v.getall(normalize(key)) def have_changed(self): """Tells whether someone has altered the headers after creation""" return self.changed def __str__(self): return str(self.v) @classmethod def from_stream(cls, stream): """Takes a stream and reads the headers, decodes headers to unicode dict like object""" return cls(parse_stream(stream)) def to_stream(self, stream): """Takes a stream and serializes headers in a mime format""" for h, v in self.v.iteritems(): try: h = h.encode('ascii') except UnicodeDecodeError: raise EncodingError("Non-ascii header name") stream.write("{0}: {1}\r\n".format(h, to_mime(h, v)))
def parse_formvars(environ, include_get_vars=True): """Parses the request, returning a MultiDict of form variables. If ``include_get_vars`` is true then GET (query string) variables will also be folded into the MultiDict. All values should be strings, except for file uploads which are left as ``FieldStorage`` instances. If the request was not a normal form request (e.g., a POST with an XML body) then ``environ['wsgi.input']`` won't be read. """ source = environ['wsgi.input'] if 'paste.parsed_formvars' in environ: parsed, check_source = environ['paste.parsed_formvars'] if check_source == source: if include_get_vars: parsed.update(parse_querystring(environ)) return parsed # @@: Shouldn't bother FieldStorage parsing during GET/HEAD and # fake_out_cgi requests type = environ.get('CONTENT_TYPE', '').lower() if ';' in type: type = type.split(';', 1)[0] fake_out_cgi = type not in ('', 'application/x-www-form-urlencoded', 'multipart/form-data') # FieldStorage assumes a default CONTENT_LENGTH of -1, but a # default of 0 is better: if not environ.get('CONTENT_LENGTH'): environ['CONTENT_LENGTH'] = '0' # Prevent FieldStorage from parsing QUERY_STRING during GET/HEAD # requests old_query_string = environ.get('QUERY_STRING','') environ['QUERY_STRING'] = '' if fake_out_cgi: input = StringIO('') old_content_type = environ.get('CONTENT_TYPE') old_content_length = environ.get('CONTENT_LENGTH') environ['CONTENT_LENGTH'] = '0' environ['CONTENT_TYPE'] = '' else: input = environ['wsgi.input'] fs = cgi.FieldStorage(fp=input, environ=environ, keep_blank_values=1) environ['QUERY_STRING'] = old_query_string if fake_out_cgi: environ['CONTENT_TYPE'] = old_content_type environ['CONTENT_LENGTH'] = old_content_length formvars = MultiDict() if isinstance(fs.value, list): for name in fs.keys(): values = fs[name] if not isinstance(values, list): values = [values] for value in values: if not value.filename: value = value.value formvars.add(name, value) environ['paste.parsed_formvars'] = (formvars, source) if include_get_vars: formvars.update(parse_querystring(environ)) return formvars
def search(self, ver=None, register=None): log.debug('search %s params: %r', register, request.params) if register == 'revision': since_time = None if 'since_id' in request.params: id = request.params['since_id'] if not id: return self._finish_bad_request( _(u'No revision specified')) rev = model.Session.query(model.Revision).get(id) if rev is None: return self._finish_not_found( _(u'There is no revision with id: %s') % id) since_time = rev.timestamp elif 'since_time' in request.params: since_time_str = request.params['since_time'] try: since_time = h.date_str_to_datetime(since_time_str) except ValueError as inst: return self._finish_bad_request('ValueError: %s' % inst) else: return self._finish_bad_request( _("Missing search term ('since_id=UUID' or " + " 'since_time=TIMESTAMP')")) revs = model.Session.query(model.Revision) \ .filter(model.Revision.timestamp > since_time) \ .order_by(model.Revision.timestamp) \ .limit(50) # reasonable enough for a page return self._finish_ok([rev.id for rev in revs]) elif register in ['dataset', 'package', 'resource']: try: params = MultiDict(self._get_search_params(request.params)) except ValueError as e: return self._finish_bad_request( _('Could not read parameters: %r' % e)) # if using API v2, default to returning the package ID if # no field list is specified if register in ['dataset', 'package'] and not params.get('fl'): params['fl'] = 'id' if ver == 2 else 'name' try: if register == 'resource': query = search.query_for(model.Resource) # resource search still uses ckan query parser options = search.QueryOptions() for k, v in params.items(): if (k in search.DEFAULT_OPTIONS.keys()): options[k] = v options.update(params) options.username = c.user options.search_tags = False options.return_objects = False query_fields = MultiDict() for field, value in params.items(): field = field.strip() if field in search.DEFAULT_OPTIONS.keys() or \ field in IGNORE_FIELDS: continue values = [value] if isinstance(value, list): values = value for v in values: query_fields.add(field, v) results = query.run( query=params.get('q'), fields=query_fields, options=options ) else: # For package searches in API v3 and higher, we can pass # parameters straight to Solr. if ver in [1, 2]: # Otherwise, put all unrecognised ones into the q # parameter params = search.\ convert_legacy_parameters_to_solr(params) query = search.query_for(model.Package) # Remove any existing fq param and set the capacity to # public if 'fq' in params: del params['fq'] params['fq'] = '+capacity:public' # if callback is specified we do not want to send that to # the search if 'callback' in params: del params['callback'] results = query.run(params) return self._finish_ok(results) except search.SearchError as e: log.exception(e) return self._finish_bad_request( _('Bad search option: %s') % e) else: return self._finish_not_found( _('Unknown register: %s') % register)
def parse_formvars(environ, include_get_vars=True): """Parses the request, returning a MultiDict of form variables. If ``include_get_vars`` is true then GET (query string) variables will also be folded into the MultiDict. All values should be strings, except for file uploads which are left as ``FieldStorage`` instances. If the request was not a normal form request (e.g., a POST with an XML body) then ``environ['wsgi.input']`` won't be read. """ source = environ['wsgi.input'] if 'paste.parsed_formvars' in environ: parsed, check_source = environ['paste.parsed_formvars'] if check_source == source: if include_get_vars: parsed.update(parse_querystring(environ)) return parsed # @@: Shouldn't bother FieldStorage parsing during GET/HEAD and # fake_out_cgi requests type = environ.get('CONTENT_TYPE', '').lower() if ';' in type: type = type.split(';', 1)[0] fake_out_cgi = type not in ('', 'application/x-www-form-urlencoded', 'multipart/form-data') # FieldStorage assumes a default CONTENT_LENGTH of -1, but a # default of 0 is better: if not environ.get('CONTENT_LENGTH'): environ['CONTENT_LENGTH'] = '0' # Prevent FieldStorage from parsing QUERY_STRING during GET/HEAD # requests old_query_string = environ.get('QUERY_STRING','') environ['QUERY_STRING'] = '' if fake_out_cgi: input = six.BytesIO(b'') old_content_type = environ.get('CONTENT_TYPE') old_content_length = environ.get('CONTENT_LENGTH') environ['CONTENT_LENGTH'] = '0' environ['CONTENT_TYPE'] = '' else: input = environ['wsgi.input'] fs = cgi.FieldStorage(fp=input, environ=environ, keep_blank_values=1) environ['QUERY_STRING'] = old_query_string if fake_out_cgi: environ['CONTENT_TYPE'] = old_content_type environ['CONTENT_LENGTH'] = old_content_length formvars = MultiDict() if isinstance(fs.value, list): for name in fs.keys(): values = fs[name] if not isinstance(values, list): values = [values] for value in values: if not value.filename: value = value.value formvars.add(name, value) environ['paste.parsed_formvars'] = (formvars, source) if include_get_vars: formvars.update(parse_querystring(environ)) return formvars
class QueryParser(object): """ The query parser will take any incoming query specifications and turn them into field-specific and general query parts. """ def __init__(self, query, terms, fields): self._query = query self._terms = terms self._fields = MultiDict(fields) @property def query(self): if not hasattr(self, '_combined_query'): parts = [self._query if self._query is not None else ''] for term in self._terms: if term.find(u' ') != -1: term = u"\"%s\"" % term parts.append(term.strip()) for field, value in self._fields.items(): if value.find(' ') != -1: value = u"\"%s\"" % value parts.append(u"%s:%s" % (field.strip(), value.strip())) self._combined_query = u' '.join(parts) return self._combined_query def _query_tokens(self): """ Split the query string, leaving quoted strings intact. """ if self._query: inside_quote = False buf = u'' for ch in self._query: if ch == u' ' and not inside_quote: if len(buf): yield buf.strip() buf = u'' elif ch == inside_quote: inside_quote = False elif ch in [u"\"", u"'"]: inside_quote = ch else: buf += ch if len(buf): yield buf.strip() def _parse_query(self): """ Decompose the query string into fields and terms. """ self._combined_fields = MultiDict(self._fields) self._combined_terms = list(self._terms) for token in self._query_tokens(): colon_pos = token.find(u':') if colon_pos != -1: field = token[:colon_pos] value = token[colon_pos+1:] value = value.strip('"').strip("'").strip() self._combined_fields.add(field, value) else: self._combined_terms.append(token) @property def fields(self): if not hasattr(self, '_combined_fields'): self._parse_query() return self._combined_fields @property def terms(self): if not hasattr(self, '_combined_terms'): self._parse_query() return self._combined_terms def validate(self): """ Check that this is a valid query. """ if not len(self.query): raise SearchError("No query has been specified") def __str__(self): return self.query def __repr__(self): return "Query(%s)" % self
class MimeHeaders(object): """Dictionary-like object that preserves the order and supports multiple values for the same key, knows whether it has been changed after the creation """ def __init__(self, items=()): self._v = MultiDict([(normalize(key), remove_newlines(val)) for (key, val) in items]) self.changed = False def __getitem__(self, key): return self._v.get(normalize(key), None) def __len__(self): return len(self._v) def __iter__(self): return iter(self._v) def __contains__(self, key): return normalize(key) in self._v def __setitem__(self, key, value): self._v[normalize(key)] = remove_newlines(value) self.changed = True def __delitem__(self, key): del self._v[normalize(key)] self.changed = True def __nonzero__(self): return len(self._v) > 0 def prepend(self, key, value): self._v._items.insert(0, (normalize(key), remove_newlines(value))) self.changed = True def add(self, key, value): """Adds header without changing the existing headers with same name""" self._v.add(normalize(key), remove_newlines(value)) self.changed = True def keys(self): """ Returns the keys. (message header names) It remembers the order in which they were added, what is really important """ return self._v.keys() def transform(self, fn): """Accepts a function, getting a key, val and returning a new pair of key, val and applies the function to all header, value pairs in the message. """ changed = [False] def tracking_fn(key, val): new_key, new_val = fn(key, val) if new_val != val or new_key != key: changed[0] = True return new_key, new_val v = MultiDict(tracking_fn(key, val) for key, val in self._v.iteritems()) if changed[0]: self._v = v self.changed = True def items(self): """ Returns header,val pairs in the preserved order. """ return list(self.iteritems()) def iteritems(self): """ Returns iterator header,val pairs in the preserved order. """ return self._v.iteritems() def get(self, key, default=None): """ Returns header value (case-insensitive). """ return self._v.get(normalize(key), default) def getall(self, key): """ Returns all header values by the given header name (case-insensitive) """ return self._v.getall(normalize(key)) def have_changed(self): """Tells whether someone has altered the headers after creation""" return self.changed def __str__(self): return str(self._v) @classmethod def from_stream(cls, stream): """Takes a stream and reads the headers, decodes headers to unicode dict like object""" return cls(parse_stream(stream)) def to_stream(self, stream): """Takes a stream and serializes headers in a mime format""" for h, v in self._v.iteritems(): try: h = h.encode('ascii') except UnicodeDecodeError: raise EncodingError("Non-ascii header name") stream.write("{0}: {1}\r\n".format(h, to_mime(h, v)))
options[k] = v options.update(params) options.username = c.user options.search_tags = False options.return_objects = False query_fields = MultiDict() for field, value in params.items(): field = field.strip() if field in search.DEFAULT_OPTIONS.keys() or \ field in IGNORE_FIELDS: continue values = [value] if isinstance(value, list): values = value for v in values: query_fields.add(field, v) results = query.run( query=params.get('q'), fields=query_fields, options=options ) else: # For package searches in API v3 and higher, we can pass # parameters straight to Solr. if ver in [1, 2]: # Otherwise, put all unrecognised ones into the q # parameter params = search.\ convert_legacy_parameters_to_solr(params) query = search.query_for(model.Package)
def get_url_security(self, cluster_id, connection=None): """ Returns the security configuration of HTTP URLs. """ # What DB class to fetch depending on the string value of the security type. sec_type_db_class = { 'tech_acc': TechnicalAccount, 'basic_auth': HTTPBasicAuth, 'wss': WSSDefinition } result = MultiDict() query = http_soap_security_list(self._session, cluster_id, connection) columns = Bunch() # So ConfigDict has its data in the format it expects for c in query.statement.columns: columns[c.name] = None for item in query.all(): _info = Bunch() _info[item.soap_action] = Bunch() _info[item.soap_action].is_active = item.is_active _info[item.soap_action].transport = item.transport _info[item.soap_action].data_format = item.data_format if item.security_id: _info[item.soap_action].sec_def = Bunch() # Will raise KeyError if the DB gets somehow misconfigured. db_class = sec_type_db_class[item.sec_type] sec_def = self._session.query(db_class).\ filter(db_class.id==item.security_id).\ one() # Common things first _info[item.soap_action].sec_def.name = sec_def.name _info[item.soap_action].sec_def.password = sec_def.password _info[item.soap_action].sec_def.sec_type = item.sec_type if item.sec_type == security_def_type.tech_account: _info[item.soap_action].sec_def.salt = sec_def.salt elif item.sec_type == security_def_type.basic_auth: _info[item.soap_action].sec_def.username = sec_def.username _info[item.soap_action].sec_def.password = sec_def.password _info[item.soap_action].sec_def.realm = sec_def.realm elif item.sec_type == security_def_type.wss: _info[item.soap_action].sec_def.username = sec_def.username _info[item.soap_action].sec_def.password = sec_def.password _info[ item. soap_action].sec_def.password_type = sec_def.password_type _info[ item. soap_action].sec_def.reject_empty_nonce_creat = sec_def.reject_empty_nonce_creat _info[ item. soap_action].sec_def.reject_stale_tokens = sec_def.reject_stale_tokens _info[ item. soap_action].sec_def.reject_expiry_limit = sec_def.reject_expiry_limit _info[ item. soap_action].sec_def.nonce_freshness_time = sec_def.nonce_freshness_time else: _info[item.soap_action].sec_def = ZATO_NONE result.add(item.url_path, _info) return result, columns
def search(self, ver=None, register=None): log.debug('search %s params: %r', register, request.params) if register == 'revision': since_time = None if 'since_id' in request.params: id = request.params['since_id'] if not id: return self._finish_bad_request( _(u'No revision specified')) rev = model.Session.query(model.Revision).get(id) if rev is None: return self._finish_not_found( _(u'There is no revision with id: %s') % id) since_time = rev.timestamp elif 'since_time' in request.params: since_time_str = request.params['since_time'] try: since_time = h.date_str_to_datetime(since_time_str) except ValueError as inst: return self._finish_bad_request('ValueError: %s' % inst) else: return self._finish_bad_request( _("Missing search term ('since_id=UUID' or " + " 'since_time=TIMESTAMP')")) revs = model.Session.query(model.Revision) \ .filter(model.Revision.timestamp > since_time) \ .order_by(model.Revision.timestamp) \ .limit(50) # reasonable enough for a page return self._finish_ok([rev.id for rev in revs]) elif register in ['dataset', 'package', 'resource']: try: params = MultiDict(self._get_search_params(request.params)) except ValueError as e: return self._finish_bad_request( _('Could not read parameters: %r' % e)) # if using API v2, default to returning the package ID if # no field list is specified if register in ['dataset', 'package'] and not params.get('fl'): params['fl'] = 'id' if ver == 2 else 'name' try: if register == 'resource': query = search.query_for(model.Resource) # resource search still uses ckan query parser options = search.QueryOptions() for k, v in params.items(): if (k in search.DEFAULT_OPTIONS.keys()): options[k] = v options.update(params) options.username = c.user options.search_tags = False options.return_objects = False query_fields = MultiDict() for field, value in params.items(): field = field.strip() if field in search.DEFAULT_OPTIONS.keys() or \ field in IGNORE_FIELDS: continue values = [value] if isinstance(value, list): values = value for v in values: query_fields.add(field, v) results = query.run(query=params.get('q'), fields=query_fields, options=options) else: # For package searches in API v3 and higher, we can pass # parameters straight to Solr. if ver in [1, 2]: # Otherwise, put all unrecognised ones into the q # parameter params = search.\ convert_legacy_parameters_to_solr(params) query = search.query_for(model.Package) # Remove any existing fq param and set the capacity to # public if 'fq' in params: del params['fq'] params['fq'] = '+capacity:public' # if callback is specified we do not want to send that to # the search if 'callback' in params: del params['callback'] results = query.run(params) return self._finish_ok(results) except search.SearchError as e: log.exception(e) return self._finish_bad_request(_('Bad search option: %s') % e) else: return self._finish_not_found(_('Unknown register: %s') % register)
def _after_init_accepted(self, server): if self.singleton_server: for(_, name, is_active, job_type, start_date, extra, service,\ _, weeks, days, hours, minutes, seconds, repeats, cron_definition)\ in self.odb.get_job_list(server.cluster.id): if is_active: job_data = Bunch({'name':name, 'is_active':is_active, 'job_type':job_type, 'start_date':start_date, 'extra':extra, 'service':service, 'weeks':weeks, 'days':days, 'hours':hours, 'minutes':minutes, 'seconds':seconds, 'repeats':repeats, 'cron_definition':cron_definition}) self.singleton_server.scheduler.create_edit('create', job_data) # Start the connectors only once throughout the whole cluster self._init_connectors(server) # Mapping between SOAP actions and internal services. #for soap_action, service_name in self.odb.get_internal_channel_list(server.cluster.id): # self.request_handler.soap_handler.soap_config[soap_action] = service_name # FTP ftp_conn_params = Bunch() for item in self.odb.get_out_ftp_list(server.cluster.id): ftp_conn_params[item.name] = Bunch() ftp_conn_params[item.name].is_active = item.is_active ftp_conn_params[item.name].name = item.name ftp_conn_params[item.name].host = item.host ftp_conn_params[item.name].user = item.user ftp_conn_params[item.name].password = item.password ftp_conn_params[item.name].acct = item.acct ftp_conn_params[item.name].timeout = item.timeout ftp_conn_params[item.name].port = item.port ftp_conn_params[item.name].dircache = item.dircache self.ftp = FTPFacade(ftp_conn_params) self.worker_config = Bunch() # Repo location so that AMQP subprocesses know how where to read # the server's configuration from. self.worker_config.repo_location = self.repo_location # The broker client for each of the worker threads. self.worker_config.broker_config = Bunch() self.worker_config.broker_config.name = 'worker-thread' self.worker_config.broker_config.broker_token = self.broker_token self.worker_config.broker_config.zmq_context = self.zmq_context self.worker_config.broker_config.broker_push_client_pull = self.broker_push_worker_pull self.worker_config.broker_config.client_push_broker_pull = self.worker_push_broker_pull self.worker_config.broker_config.broker_pub_client_sub = self.broker_pub_worker_sub # HTTP Basic Auth ba_config = Bunch() for item in self.odb.get_basic_auth_list(server.cluster.id): ba_config[item.name] = Bunch() ba_config[item.name].is_active = item.is_active ba_config[item.name].username = item.username ba_config[item.name].domain = item.domain ba_config[item.name].password = item.password # Technical accounts ta_config = Bunch() for item in self.odb.get_tech_acc_list(server.cluster.id): ta_config[item.name] = Bunch() ta_config[item.name].is_active = item.is_active ta_config[item.name].name = item.name ta_config[item.name].password = item.password ta_config[item.name].salt = item.salt wss_config = Bunch() for item in self.odb.get_wss_list(server.cluster.id): wss_config[item.name] = Bunch() wss_config[item.name].is_active = item.is_active wss_config[item.name].username = item.username wss_config[item.name].password = item.password wss_config[item.name].password_type = item.password_type wss_config[item.name].reject_empty_nonce_ts = item.reject_empty_nonce_ts wss_config[item.name].reject_stale_username = item.reject_stale_username wss_config[item.name].expiry_limit = item.expiry_limit wss_config[item.name].nonce_freshness = item.nonce_freshness # Security configuration of HTTP URLs. url_sec = self.odb.get_url_security(server) # All the HTTP/SOAP channels. http_soap = MultiDict() for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'): _info = Bunch() _info[item.soap_action] = Bunch() _info[item.soap_action].id = item.id _info[item.soap_action].name = item.name _info[item.soap_action].is_internal = item.is_internal _info[item.soap_action].url_path = item.url_path _info[item.soap_action].method = item.method _info[item.soap_action].soap_version = item.soap_version _info[item.soap_action].service_id = item.service_id _info[item.soap_action].service_name = item.service_name _info[item.soap_action].impl_name = item.impl_name http_soap.add(item.url_path, _info) self.worker_config.basic_auth = ba_config self.worker_config.tech_acc = ta_config self.worker_config.wss = wss_config self.worker_config.url_sec = url_sec self.worker_config.http_soap = http_soap # The parallel server's broker client. The client's used to notify # all the server's AMQP subprocesses that they need to shut down. self.broker_client = BrokerClient() self.broker_client.name = 'parallel' self.broker_client.token = server.cluster.broker_token self.broker_client.zmq_context = self.zmq_context self.broker_client.client_push_broker_pull = self.parallel_push_broker_pull self.broker_client.init() self.broker_client.start()
class QueryParser(object): """ The query parser will take any incoming query specifications and turn them into field-specific and general query parts. """ def __init__(self, query, terms, fields): self._query = query self._terms = terms self._fields = MultiDict(fields) @property def query(self): if not hasattr(self, '_combined_query'): parts = [self._query if self._query is not None else ''] for term in self._terms: if term.find(u' ') != -1: term = u"\"%s\"" % term parts.append(term.strip()) for field, value in self._fields.items(): if value.find(' ') != -1: value = u"\"%s\"" % value parts.append(u"%s:%s" % (field.strip(), value.strip())) self._combined_query = u' '.join(parts) return self._combined_query def _query_tokens(self): """ Split the query string, leaving quoted strings intact. """ if self._query: inside_quote = False buf = u'' for ch in self._query: if ch == u' ' and not inside_quote: if len(buf): yield buf.strip() buf = u'' elif ch == inside_quote: inside_quote = False elif ch in [u"\"", u"'"]: inside_quote = ch else: buf += ch if len(buf): yield buf.strip() def _parse_query(self): """ Decompose the query string into fields and terms. """ self._combined_fields = MultiDict(self._fields) self._combined_terms = list(self._terms) for token in self._query_tokens(): colon_pos = token.find(u':') if colon_pos != -1: field = token[:colon_pos] value = token[colon_pos + 1:] value = value.strip('"').strip("'").strip() self._combined_fields.add(field, value) else: self._combined_terms.append(token) @property def fields(self): if not hasattr(self, '_combined_fields'): self._parse_query() return self._combined_fields @property def terms(self): if not hasattr(self, '_combined_terms'): self._parse_query() return self._combined_terms def validate(self): """ Check that this is a valid query. """ pass def __str__(self): return self.query def __repr__(self): return "Query(%r)" % self.query
if k in search.DEFAULT_OPTIONS.keys(): options[k] = v options.update(params) options.username = c.user options.search_tags = False options.return_objects = False query_fields = MultiDict() for field, value in params.items(): field = field.strip() if field in search.DEFAULT_OPTIONS.keys() or field in IGNORE_FIELDS: continue values = [value] if isinstance(value, list): values = value for v in values: query_fields.add(field, v) results = query.run(query=params.get("q"), fields=query_fields, options=options) else: # For package searches in API v3 and higher, we can pass # parameters straight to Solr. if ver in [1, 2]: # Otherwise, put all unrecognised ones into the q # parameter params = search.convert_legacy_parameters_to_solr(params) query = search.query_for(model.Package) # Remove any existing fq param and set the capacity to # public if "fq" in params: del params["fq"]
mainkey = '' d = MultiDict() DEBUG = 0 for item in items: cleanitem = item thiskey = item.find('b') #, attrs={'img':'class bdtIcon'}) if thiskey: value = str(item) value = value.replace(str(thiskey), '') value = cleaner(value) if DEBUG: print "X %s" % value mainkey = thiskey if value: d.add(thiskey.text, value) else: if mainkey: if DEBUG: print mainkey.text print item d.add(mainkey.text, item) # final result shown = {} xmlfile.write("<?xml version=\"1.0\"?>\n") xmlfile.write("<entry xmlns=\"http://www.w3.org/2005/Atom\"\n") xmlfile.write(" xmlns:dcterms=\"http://purl.org/dc/terms/\">\n") xmlfile.write(" <dcterms:source>%s</dcterms:source>\n" % inturl)
def get_url_security(self, cluster_id): """ Returns the security configuration of HTTP URLs. """ # What DB class to fetch depending on the string value of the security type. sec_type_db_class = { 'tech_acc': TechnicalAccount, 'basic_auth': HTTPBasicAuth, 'wss': WSSDefinition } result = MultiDict() query = http_soap_security_list(self._session, cluster_id) columns = Bunch() # So ConfigDict has its data in the format it expects for c in query.statement.columns: columns[c.name] = None for item in query.all(): _info = Bunch() _info[item.soap_action] = Bunch() _info[item.soap_action].transport = item.transport _info[item.soap_action].data_format = item.data_format if item.security_id: _info[item.soap_action].sec_def = Bunch() # Will raise KeyError if the DB gets somehow misconfigured. db_class = sec_type_db_class[item.sec_type] sec_def = self._session.query(db_class).\ filter(db_class.id==item.security_id).\ one() # Common things first _info[item.soap_action].sec_def.name = sec_def.name _info[item.soap_action].sec_def.password = sec_def.password _info[item.soap_action].sec_def.sec_type = item.sec_type if item.sec_type == security_def_type.tech_account: _info[item.soap_action].sec_def.salt = sec_def.salt elif item.sec_type == security_def_type.basic_auth: _info[item.soap_action].sec_def.username = sec_def.username _info[item.soap_action].sec_def.password = sec_def.password _info[item.soap_action].sec_def.realm = sec_def.realm elif item.sec_type == security_def_type.wss: _info[item.soap_action].sec_def.username = sec_def.username _info[item.soap_action].sec_def.password = sec_def.password _info[item.soap_action].sec_def.password_type = sec_def.password_type _info[item.soap_action].sec_def.reject_empty_nonce_creat = sec_def.reject_empty_nonce_creat _info[item.soap_action].sec_def.reject_stale_tokens = sec_def.reject_stale_tokens _info[item.soap_action].sec_def.reject_expiry_limit = sec_def.reject_expiry_limit _info[item.soap_action].sec_def.nonce_freshness_time = sec_def.nonce_freshness_time else: _info[item.soap_action].sec_def = ZATO_NONE result.add(item.url_path, _info) return result, columns