def init(self): plain_http_config = MultiDict() soap_config = MultiDict() dol = deepcopy(self.worker_config.http_soap).dict_of_lists() for url_path in dol: for item in dol[url_path]: for soap_action, channel_info in item.items(): if channel_info['connection'] == 'channel': if channel_info.transport == 'plain_http': config = plain_http_config.setdefault(url_path, Bunch()) config[soap_action] = deepcopy(channel_info) else: config = soap_config.setdefault(url_path, Bunch()) config[soap_action] = deepcopy(channel_info) self.request_dispatcher = RequestDispatcher(simple_io_config=self.worker_config.simple_io) self.request_dispatcher.soap_handler = SOAPHandler(soap_config, self.server) self.request_dispatcher.plain_http_handler = PlainHTTPHandler(plain_http_config, self.server) # Statistics maintenance self.stats_maint = MaintenanceTool(self.kvdb.conn) self.request_dispatcher.security = ConnectionHTTPSOAPSecurity( self.server.odb.get_url_security(self.server.cluster_id, 'channel')[0], self.worker_config.basic_auth, self.worker_config.tech_acc, self.worker_config.wss) # Create all the expected connections self.init_sql() self.init_ftp() self.init_http_soap()
def parse_dict_querystring(environ): """Parses a query string like parse_querystring, but returns a MultiDict Caches this value in case parse_dict_querystring is called again for the same request. Example:: >>> environ = {'QUERY_STRING': 'day=Monday&user=fred&user=jane'} >>> parsed = parse_dict_querystring(environ) >>> parsed['day'] 'Monday' >>> parsed['user'] 'fred' >>> parsed.getall('user') ['fred', 'jane'] """ source = environ.get('QUERY_STRING', '') if not source: return MultiDict() if 'paste.parsed_dict_querystring' in environ: parsed, check_source = environ['paste.parsed_dict_querystring'] if check_source == source: return parsed parsed = parse_qsl(source, keep_blank_values=True, strict_parsing=False) multi = MultiDict(parsed) environ['paste.parsed_dict_querystring'] = (multi, source) return multi
def handle(self): """NOTE: The result of this function is *backwards*. It returns None for success. Just like C. """ sub = self.subscriber params = dict(sub.params) params.update(self.data) h = httplib2.Http() h.follow_all_redirects = sub.follow_all_redirects headers = MultiDict(sub.headers) body = sub.body if sub.username: h.add_credentials(sub.username, sub.password) params = dict((key, simplejson.dumps(value)) for key, value in params.items()) if sub.method == "GET": #merge params with query string qs = sub.queryString if qs: qs += "&" else: qs = "?" qs += urllib.urlencode(params) headers['Content-Length'] = '0' else: body = urllib.urlencode(params) headers['Content-Length'] = str(len(body)) if not headers.has_key('Content-Type'): headers['Content-Type'] = 'application/x-www-form-urlencoded' username = config.get('username', None) password = config.get('password', None) if username: headers['AUTHORIZATION'] = 'WSSE profile="UsernameToken"' headers['X_WSSE'] = wsse_header(username, password) __traceback_info__ = '%s %s (%i bytes in body)' % (sub.method, self.url, len(body)) log.info('Sending event %s %s (%i bytes in body, id=%s' % (self.url, sub.method, len(body), self.id)) self.last_sent = datetime.now() self.failures += 1 #if we succeed, we won't care that someone called #us a failure if self.failures > MAX_SEND_FAILURES: self.fail() return None try: response = h.request(self.url, method=sub.method, body=body, headers=headers, redirections=sub.redirections) except socket.error, e: print >> sys.stderr, 'Error doing %s %s (body length: %i bytes)' % (self.url, sub.method, len(body)) return "%s" % e, ''
def search(self, ver=None, register=None): id = None try: params = MultiDict(self._get_search_params(request.params)) if 'q' in params.keys(): id = params['q'] if 'query' in params.keys(): id = params['query'] except ValueError, e: pass
def parse_path(self): # parse path and parameters path = self.path.split('?')[0] params = self.path[len(path) + 1:].split('&') param_dict = MultiDict() for param in params: key = param.split('=')[0] value = param[len(key) + 1:] param_dict.add(key, value) return path, param_dict
def parse_path(self): # parse path and parameters path = self.path.split('?')[0] params = self.path[len(path)+1:].split('&') param_dict = MultiDict() for param in params: key = param.split('=')[0] value = param[len(key)+1:] param_dict.add(key, value) return path, param_dict
def search(self, ver=None, register=None): id = None try: params = MultiDict(self._get_search_params(request.params)) if 'q' in params.keys(): id = params['q'] if 'query' in params.keys(): id = params['query'] except ValueError, e: log.debug(str(e)) pass
def _parse_query(self): """ Decompose the query string into fields and terms. """ self._combined_fields = MultiDict(self._fields) self._combined_terms = list(self._terms) for token in self._query_tokens(): colon_pos = token.find(u':') if colon_pos != -1: field = token[:colon_pos] value = token[colon_pos + 1:] value = value.strip('"').strip("'").strip() self._combined_fields.add(field, value) else: self._combined_terms.append(token)
def search(self, ver=None, register=None): id = None try: params = MultiDict(self._get_search_params(request.params)) if "q" in list(params.keys()): id = params["q"] if "query" in list(params.keys()): id = params["query"] except ValueError as e: log.debug(str(e)) pass self._post_analytics(c.user, register, "search", id) return ApiController.search(self, ver, register)
def __init__(self, log, location, urls=None, avgs=[1, 7], past_only=False, config=None): super(GoalAnalyser, self).__init__() self.config = config self.log = log if urls is None: if self.config is None: self.urls = {} else: self.urls = config.urls() else: self.urls = urls self.avgs = avgs self.dir = location self.past_only = past_only self.statscounters = MultiDict() self.outputs = {} self.files = {} self.existing_dates = {} self.parse = logparser() self.log_entries = {} if self.config is not None: for section in self.config.sections(): log = dict(self.config.items(section)).get('log', '').lower() if log in ('true', 'yes', 'on'): fn = '%s.log' % section self.log_entries[section] = fn self.files[fn] = open(os.path.join(self.dir, fn), 'w')
def test_parsevars(): e = make_post('a=1&b=2&c=3&b=4') cur_input = e['wsgi.input'] d = parse_formvars(e) assert isinstance(d, MultiDict) assert d == MultiDict([('a', '1'), ('b', '2'), ('c', '3'), ('b', '4')]) assert e['paste.parsed_formvars'] == ((d, e['wsgi.input']))
def copy(self): """ Creates a copy of this ConfigStore. All configuration data is copied over except for SQL connections. """ config_store = ConfigStore() # Grab all ConfigDicts - even if they're actually ZATO_NONE - and make their copies for attr_name in dir(self): attr = getattr(self, attr_name) if isinstance(attr, ConfigDict): copy_func = getattr(attr, 'copy') setattr(config_store, attr_name, copy_func()) elif attr is ZATO_NONE: setattr(config_store, attr_name, ZATO_NONE) http_soap = MultiDict() dict_of_lists = self.http_soap.dict_of_lists() for url_path, lists in dict_of_lists.items(): _info = Bunch() for elem in lists: for soap_action, item in elem.items(): _info[soap_action] = Bunch() _info[soap_action].id = item.id _info[soap_action].name = item.name _info[soap_action].is_active = item.is_active _info[soap_action].is_internal = item.is_internal _info[soap_action].url_path = item.url_path _info[soap_action].method = item.method _info[soap_action].soap_version = item.soap_version _info[soap_action].service_id = item.service_id _info[soap_action].service_name = item.service_name _info[soap_action].impl_name = item.impl_name _info[soap_action].transport = item.transport _info[soap_action].connection = item.connection http_soap.add(url_path, _info) config_store.http_soap = http_soap config_store.url_sec = self.url_sec config_store.broker_config = self.broker_config config_store.odb_data = deepcopy(self.odb_data) return config_store
def _get_params(self, request): ''' Returns the parameter from request, regardless their type (GET OR POST) and returns a MultiDict containg the values. ''' params = MultiDict() if request.method == 'POST': # this is required by aiohttp. without this, # request.POST is empty _ = [f for f in request.post()] for k, v in request.POST.items(): # when data is transfered via POST, array names are # appended a '[]'. params.add(k.replace('[]', ''), v) elif request.method == 'GET': for k, v in request.GET.items(): params.add(k, v) else: raise TypeError('You need to pass a Request obj') return params
def test_publish_with_multiple_urls(self, mock): data = MultiDict({'hub.mode': 'publish'}) data.add('hub.url', 'http://www.example.com/') data.add('hub.url', 'http://www.site.com/') request = self.r('/publish', self.valid_headers, POST=data) info = publish(None, request) self.assertEqual(info.status_code, 204)
def _parse_query(self): """ Decompose the query string into fields and terms. """ self._combined_fields = MultiDict(self._fields) self._combined_terms = list(self._terms) for token in self._query_tokens(): colon_pos = token.find(u':') if colon_pos != -1: field = token[:colon_pos] value = token[colon_pos+1:] value = value.strip('"').strip("'").strip() self._combined_fields.add(field, value) else: self._combined_terms.append(token)
def test_view_login(self): from apex.lib.libapex import create_user create_user(username='******', password='******') from apex.views import login request = testing.DummyRequest() # wtforms requires this request.POST = MultiDict() request.context = testing.DummyResource() response = login(request) self.assertEqual(response['title'], 'You need to login')
def _external_request(self, uri, method, body, headers, wsgi_request, input, output, trusted): body = self._serialize_body(input, body, headers) # @@: Does httplib2 handle Content-Length? dict_headers = MultiDict(headers) (res, content) = self.httplib2.request( uri, method=method, body=body, headers=dict_headers, redirections=self.redirections) status = '%s %s' % (res.status, res.reason) # @@: Hrm... headers = res.items() remove_header(headers, 'status') return self._create_response( status, headers, output, [content], trusted)
def frontpage_featured_orgs(self, path=None, data=None, errors=None, error_summary=None): msg = '' if p.toolkit.request.method == 'POST' and not data: data = MultiDict(p.toolkit.request.POST) data = data.getall('featured_orgs') if len(data) < 1: msg = 'At least one featured org must be selected' h.flash_error(msg, allow_html=False) else: forgs = '' for i in range(len(data)): data[i] = data[i].encode('utf-8') forgs += data[i] if i < len(data) - 1: forgs += ' ' try: junk = p.toolkit.get_action('config_option_update')( { 'user': c.user }, { 'ckan.featured_orgs': forgs }) except p.toolkit.ValidationError, e: errors = e.error_dict error_summary = e.error_summary return self.frontpage_featured_orgs( '', data, errors, error_summary) p.toolkit.redirect_to('/frontpage')
def test_fail_login(self): from apex.lib.libapex import create_user create_user(username='******', password='******') request = testing.DummyRequest(environ=environ) request.method = 'POST' # wtforms requires this request.POST = MultiDict() request.POST['username'] = '******' request.POST['password'] = '******' from apex.views import login request.context = testing.DummyResource() response = login(request) self.assertEqual(len(response['form'].errors), 1)
def transform(self, fn): """Accepts a function, getting a key, val and returning a new pair of key, val and applies the function to all header, value pairs in the message. """ changed = [False] def tracking_fn(key, val): new_key, new_val = fn(key, val) if new_val != val or new_key != key: changed[0] = True return new_key, new_val v = MultiDict(tracking_fn(key, val) for key, val in self.v.iteritems()) if changed[0]: self.v = v self.changed = True
def _loadOwsParams(self): """ Method to load OWS parameters from the query string. If request method is POST then this is ignored after creating the instance dictionary self._owsParams. """ # All OWS parameter names are case insensitive. self._owsParams = {} if self._requestMethod() == "POST": return log.debug('REQUEST: %s' % request) if PARAMETER_MODE == 'html_4': params = request.params elif PARAMETER_MODE == 'wps_1': # request.params will use ';' as a QS separator which is not compatible # with WPS-1.0. Therefore we parse the QS ourselves. qs = request.environ['QUERY_STRING'] params = MultiDict( parse_qsl(qs, semicolon_separator=False, keep_blank_values=True, strict_parsing=False)) else: raise ValueError( "Value of cows.parameter_mode not recognised (%s)" % PARAMETER_MODE) #!TODO: unicode is converted here. # At some point we need to expect COWS apps to accept unicode try: for k in params: if k.lower() == 'x': self._owsParams['i'] = str(params[k]) elif k.lower() == 'y': self._owsParams['j'] = str(params[k]) else: self._owsParams[k.lower()] = str(params[k]) except UnicodeError: raise ValueError( "Cannot convert unicode to string. COWS does not accept unicode parameters" )
def test_publish_fetches_topic_content(self, mock): data = MultiDict({'hub.mode': 'publish'}) data.add('hub.url', 'http://www.example.com/') data.add('hub.url', 'http://www.site.com/') request = self.r('/publish', self.valid_headers, POST=data) hub = request.root info = publish(None, request) first = hub.topics.get('http://www.example.com/') second = hub.topics.get('http://www.site.com/') self.assertEqual(info.status_code, 204) self.assertTrue(first.timestamp is not None) self.assertTrue(second.timestamp is not None)
def params(self): """Dictionary-like object of keys from POST, GET, URL dicts Return a key value from the parameters, they are checked in the following order: POST, GET, URL Additional methods supported: ``getlist(key)`` Returns a list of all the values by that key, collected from POST, GET, URL dicts Returns a ``MultiDict`` container or a ``UnicodeMultiDict`` when ``charset`` is set. """ params = MultiDict() params.update(self._POST()) params.update(self._GET()) if self.charset: params = UnicodeMultiDict(params, encoding=self.charset, errors=self.errors, decode_keys=self.decode_param_names) return params
class MimeHeaders(object): """Dictionary-like object that preserves the order and supports multiple values for the same key, knows whether it has been changed after the creation """ def __init__(self, items=()): self._v = MultiDict([(normalize(key), remove_newlines(val)) for (key, val) in items]) self.changed = False def __getitem__(self, key): return self._v.get(normalize(key), None) def __len__(self): return len(self._v) def __iter__(self): return iter(self._v) def __contains__(self, key): return normalize(key) in self._v def __setitem__(self, key, value): self._v[normalize(key)] = remove_newlines(value) self.changed = True def __delitem__(self, key): del self._v[normalize(key)] self.changed = True def __nonzero__(self): return len(self._v) > 0 def prepend(self, key, value): self._v._items.insert(0, (normalize(key), remove_newlines(value))) self.changed = True def add(self, key, value): """Adds header without changing the existing headers with same name""" self._v.add(normalize(key), remove_newlines(value)) self.changed = True def keys(self): """ Returns the keys. (message header names) It remembers the order in which they were added, what is really important """ return self._v.keys() def transform(self, fn): """Accepts a function, getting a key, val and returning a new pair of key, val and applies the function to all header, value pairs in the message. """ changed = [False] def tracking_fn(key, val): new_key, new_val = fn(key, val) if new_val != val or new_key != key: changed[0] = True return new_key, new_val v = MultiDict(tracking_fn(key, val) for key, val in self._v.iteritems()) if changed[0]: self._v = v self.changed = True def items(self): """ Returns header,val pairs in the preserved order. """ return list(self.iteritems()) def iteritems(self): """ Returns iterator header,val pairs in the preserved order. """ return self._v.iteritems() def get(self, key, default=None): """ Returns header value (case-insensitive). """ return self._v.get(normalize(key), default) def getall(self, key): """ Returns all header values by the given header name (case-insensitive) """ return self._v.getall(normalize(key)) def have_changed(self): """Tells whether someone has altered the headers after creation""" return self.changed def __str__(self): return str(self._v) @classmethod def from_stream(cls, stream): """Takes a stream and reads the headers, decodes headers to unicode dict like object""" return cls(parse_stream(stream)) def to_stream(self, stream): """Takes a stream and serializes headers in a mime format""" for h, v in self._v.iteritems(): try: h = h.encode('ascii') except UnicodeDecodeError: raise EncodingError("Non-ascii header name") stream.write("{0}: {1}\r\n".format(h, to_mime(h, v)))
since_time = rev.timestamp elif "since_time" in request.params: since_time_str = request.params["since_time"] try: since_time = h.date_str_to_datetime(since_time_str) except ValueError, inst: return self._finish_bad_request("ValueError: %s" % inst) else: return self._finish_bad_request( _("Missing search term ('since_id=UUID' or " + " 'since_time=TIMESTAMP')") ) revs = model.Session.query(model.Revision).filter(model.Revision.timestamp > since_time) return self._finish_ok([rev.id for rev in revs]) elif register in ["dataset", "package", "resource"]: try: params = MultiDict(self._get_search_params(request.params)) except ValueError, e: return self._finish_bad_request(_("Could not read parameters: %r" % e)) # if using API v2, default to returning the package ID if # no field list is specified if register in ["dataset", "package"] and not params.get("fl"): params["fl"] = "id" if ver == 2 else "name" try: if register == "resource": query = search.query_for(model.Resource) # resource search still uses ckan query parser options = search.QueryOptions() for k, v in params.items():
since_time_str = request.params['since_time'] try: since_time = h.date_str_to_datetime(since_time_str) except ValueError, inst: return self._finish_bad_request('ValueError: %s' % inst) else: return self._finish_bad_request( _("Missing search term ('since_id=UUID' or " + " 'since_time=TIMESTAMP')")) revs = model.Session.query(model.Revision).\ filter(model.Revision.timestamp > since_time).\ limit(50) # reasonable enough for a page return self._finish_ok([rev.id for rev in revs]) elif register in ['dataset', 'package', 'resource']: try: params = MultiDict(self._get_search_params(request.params)) except ValueError, e: return self._finish_bad_request( _('Could not read parameters: %r' % e)) # if using API v2, default to returning the package ID if # no field list is specified if register in ['dataset', 'package'] and not params.get('fl'): params['fl'] = 'id' if ver == 2 else 'name' try: if register == 'resource': query = search.query_for(model.Resource) # resource search still uses ckan query parser options = search.QueryOptions()
def __init__(self, items=()): self._v = MultiDict([(normalize(key), remove_newlines(val)) for (key, val) in items]) self.changed = False
def search(self, ver=None, register=None): log.debug('search %s params: %r', register, request.params) if register == 'revision': since_time = None if 'since_id' in request.params: id = request.params['since_id'] if not id: return self._finish_bad_request( _(u'No revision specified')) rev = model.Session.query(model.Revision).get(id) if rev is None: return self._finish_not_found( _(u'There is no revision with id: %s') % id) since_time = rev.timestamp elif 'since_time' in request.params: since_time_str = request.params['since_time'] try: since_time = h.date_str_to_datetime(since_time_str) except ValueError as inst: return self._finish_bad_request('ValueError: %s' % inst) else: return self._finish_bad_request( _("Missing search term ('since_id=UUID' or " + " 'since_time=TIMESTAMP')")) revs = model.Session.query(model.Revision) \ .filter(model.Revision.timestamp > since_time) \ .order_by(model.Revision.timestamp) \ .limit(50) # reasonable enough for a page return self._finish_ok([rev.id for rev in revs]) elif register in ['dataset', 'package', 'resource']: try: params = MultiDict(self._get_search_params(request.params)) except ValueError as e: return self._finish_bad_request( _('Could not read parameters: %r' % e)) # if using API v2, default to returning the package ID if # no field list is specified if register in ['dataset', 'package'] and not params.get('fl'): params['fl'] = 'id' if ver == 2 else 'name' try: if register == 'resource': query = search.query_for(model.Resource) # resource search still uses ckan query parser options = search.QueryOptions() for k, v in params.items(): if (k in search.DEFAULT_OPTIONS.keys()): options[k] = v options.update(params) options.username = c.user options.search_tags = False options.return_objects = False query_fields = MultiDict() for field, value in params.items(): field = field.strip() if field in search.DEFAULT_OPTIONS.keys() or \ field in IGNORE_FIELDS: continue values = [value] if isinstance(value, list): values = value for v in values: query_fields.add(field, v) results = query.run( query=params.get('q'), fields=query_fields, options=options ) else: # For package searches in API v3 and higher, we can pass # parameters straight to Solr. if ver in [1, 2]: # Otherwise, put all unrecognised ones into the q # parameter params = search.\ convert_legacy_parameters_to_solr(params) query = search.query_for(model.Package) # Remove any existing fq param and set the capacity to # public if 'fq' in params: del params['fq'] params['fq'] = '+capacity:public' # if callback is specified we do not want to send that to # the search if 'callback' in params: del params['callback'] results = query.run(params) return self._finish_ok(results) except search.SearchError as e: log.exception(e) return self._finish_bad_request( _('Bad search option: %s') % e) else: return self._finish_not_found( _('Unknown register: %s') % register)
class QueryParser(object): """ The query parser will take any incoming query specifications and turn them into field-specific and general query parts. """ def __init__(self, query, terms, fields): self._query = query self._terms = terms self._fields = MultiDict(fields) @property def query(self): if not hasattr(self, '_combined_query'): parts = [self._query if self._query is not None else ''] for term in self._terms: if term.find(u' ') != -1: term = u"\"%s\"" % term parts.append(term.strip()) for field, value in self._fields.items(): if value.find(' ') != -1: value = u"\"%s\"" % value parts.append(u"%s:%s" % (field.strip(), value.strip())) self._combined_query = u' '.join(parts) return self._combined_query def _query_tokens(self): """ Split the query string, leaving quoted strings intact. """ if self._query: inside_quote = False buf = u'' for ch in self._query: if ch == u' ' and not inside_quote: if len(buf): yield buf.strip() buf = u'' elif ch == inside_quote: inside_quote = False elif ch in [u"\"", u"'"]: inside_quote = ch else: buf += ch if len(buf): yield buf.strip() def _parse_query(self): """ Decompose the query string into fields and terms. """ self._combined_fields = MultiDict(self._fields) self._combined_terms = list(self._terms) for token in self._query_tokens(): colon_pos = token.find(u':') if colon_pos != -1: field = token[:colon_pos] value = token[colon_pos + 1:] value = value.strip('"').strip("'").strip() self._combined_fields.add(field, value) else: self._combined_terms.append(token) @property def fields(self): if not hasattr(self, '_combined_fields'): self._parse_query() return self._combined_fields @property def terms(self): if not hasattr(self, '_combined_terms'): self._parse_query() return self._combined_terms def validate(self): """ Check that this is a valid query. """ pass def __str__(self): return self.query def __repr__(self): return "Query(%r)" % self.query
def get_url_security(self, cluster_id): """ Returns the security configuration of HTTP URLs. """ # What DB class to fetch depending on the string value of the security type. sec_type_db_class = { 'tech_acc': TechnicalAccount, 'basic_auth': HTTPBasicAuth, 'wss': WSSDefinition } result = MultiDict() query = http_soap_security_list(self._session, cluster_id) columns = Bunch() # So ConfigDict has its data in the format it expects for c in query.statement.columns: columns[c.name] = None for item in query.all(): _info = Bunch() _info[item.soap_action] = Bunch() _info[item.soap_action].transport = item.transport _info[item.soap_action].data_format = item.data_format if item.security_id: _info[item.soap_action].sec_def = Bunch() # Will raise KeyError if the DB gets somehow misconfigured. db_class = sec_type_db_class[item.sec_type] sec_def = self._session.query(db_class).\ filter(db_class.id==item.security_id).\ one() # Common things first _info[item.soap_action].sec_def.name = sec_def.name _info[item.soap_action].sec_def.password = sec_def.password _info[item.soap_action].sec_def.sec_type = item.sec_type if item.sec_type == security_def_type.tech_account: _info[item.soap_action].sec_def.salt = sec_def.salt elif item.sec_type == security_def_type.basic_auth: _info[item.soap_action].sec_def.username = sec_def.username _info[item.soap_action].sec_def.password = sec_def.password _info[item.soap_action].sec_def.realm = sec_def.realm elif item.sec_type == security_def_type.wss: _info[item.soap_action].sec_def.username = sec_def.username _info[item.soap_action].sec_def.password = sec_def.password _info[item.soap_action].sec_def.password_type = sec_def.password_type _info[item.soap_action].sec_def.reject_empty_nonce_creat = sec_def.reject_empty_nonce_creat _info[item.soap_action].sec_def.reject_stale_tokens = sec_def.reject_stale_tokens _info[item.soap_action].sec_def.reject_expiry_limit = sec_def.reject_expiry_limit _info[item.soap_action].sec_def.nonce_freshness_time = sec_def.nonce_freshness_time else: _info[item.soap_action].sec_def = ZATO_NONE result.add(item.url_path, _info) return result, columns
def _test_unicode_dict(decode_param_names=False): d = UnicodeMultiDict(MultiDict({b'a': 'a test'})) d.encoding = 'utf-8' d.errors = 'ignore' if decode_param_names: key_str = six.text_type k = lambda key: key d.decode_keys = True else: key_str = six.binary_type k = lambda key: key.encode() def assert_unicode(obj): assert isinstance(obj, six.text_type) def assert_key_str(obj): assert isinstance(obj, key_str) def assert_unicode_item(obj): key, value = obj assert isinstance(key, key_str) assert isinstance(value, six.text_type) assert d.items() == [(k('a'), u'a test')] map(assert_key_str, d.keys()) map(assert_unicode, d.values()) d[b'b'] = b'2 test' d[b'c'] = b'3 test' assert d.items() == [(k('a'), u'a test'), (k('b'), u'2 test'), (k('c'), u'3 test')] list(map(assert_unicode_item, d.items())) d[k('b')] = b'4 test' assert d.items() == [(k('a'), u'a test'), (k('c'), u'3 test'), (k('b'), u'4 test')], d.items() list(map(assert_unicode_item, d.items())) d.add(k('b'), b'5 test') pytest.raises(KeyError, d.getone, k("b")) assert d.getall(k('b')) == [u'4 test', u'5 test'] map(assert_unicode, d.getall('b')) assert d.items() == [(k('a'), u'a test'), (k('c'), u'3 test'), (k('b'), u'4 test'), (k('b'), u'5 test')] list(map(assert_unicode_item, d.items())) del d[k('b')] assert d.items() == [(k('a'), u'a test'), (k('c'), u'3 test')] list(map(assert_unicode_item, d.items())) assert d.pop('xxx', u'5 test') == u'5 test' assert isinstance(d.pop('xxx', u'5 test'), six.text_type) assert d.getone(k('a')) == u'a test' assert isinstance(d.getone(k('a')), six.text_type) assert d.popitem() == (k('c'), u'3 test') d[k('c')] = b'3 test' assert_unicode_item(d.popitem()) assert d.items() == [(k('a'), u'a test')] list(map(assert_unicode_item, d.items())) item = [] assert d.setdefault(k('z'), item) is item items = d.items() assert items == [(k('a'), u'a test'), (k('z'), item)] assert isinstance(items[1][0], key_str) assert isinstance(items[1][1], list) assert isinstance(d.setdefault(k('y'), b'y test'), six.text_type) assert isinstance(d[k('y')], six.text_type) assert d.mixed() == {k('a'): u'a test', k('y'): u'y test', k('z'): item} assert d.dict_of_lists() == {k('a'): [u'a test'], k('y'): [u'y test'], k('z'): [item]} del d[k('z')] list(map(assert_unicode_item, six.iteritems(d.mixed()))) list(map(assert_unicode_item, [(key, value[0]) for \ key, value in six.iteritems(d.dict_of_lists())])) assert k('a') in d dcopy = d.copy() assert dcopy is not d assert dcopy == d d[k('x')] = 'x test' assert dcopy != d d[(1, None)] = (None, 1) assert d.items() == [(k('a'), u'a test'), (k('y'), u'y test'), (k('x'), u'x test'), ((1, None), (None, 1))] item = d.items()[-1] assert isinstance(item[0], tuple) assert isinstance(item[1], tuple) fs = cgi.FieldStorage() fs.name = 'thefile' fs.filename = 'hello.txt' fs.file = StringIO('hello') d[k('f')] = fs ufs = d[k('f')] assert isinstance(ufs, cgi.FieldStorage) assert ufs is not fs assert ufs.name == fs.name assert isinstance(ufs.name, str if six.PY3 else key_str) assert ufs.filename == fs.filename assert isinstance(ufs.filename, six.text_type) assert isinstance(ufs.value, str) assert ufs.value == 'hello'
def _after_init_accepted(self, server): if self.singleton_server: for(_, name, is_active, job_type, start_date, extra, service,\ _, weeks, days, hours, minutes, seconds, repeats, cron_definition)\ in self.odb.get_job_list(server.cluster.id): if is_active: job_data = Bunch({'name':name, 'is_active':is_active, 'job_type':job_type, 'start_date':start_date, 'extra':extra, 'service':service, 'weeks':weeks, 'days':days, 'hours':hours, 'minutes':minutes, 'seconds':seconds, 'repeats':repeats, 'cron_definition':cron_definition}) self.singleton_server.scheduler.create_edit('create', job_data) # Start the connectors only once throughout the whole cluster self._init_connectors(server) # Mapping between SOAP actions and internal services. #for soap_action, service_name in self.odb.get_internal_channel_list(server.cluster.id): # self.request_handler.soap_handler.soap_config[soap_action] = service_name # FTP ftp_conn_params = Bunch() for item in self.odb.get_out_ftp_list(server.cluster.id): ftp_conn_params[item.name] = Bunch() ftp_conn_params[item.name].is_active = item.is_active ftp_conn_params[item.name].name = item.name ftp_conn_params[item.name].host = item.host ftp_conn_params[item.name].user = item.user ftp_conn_params[item.name].password = item.password ftp_conn_params[item.name].acct = item.acct ftp_conn_params[item.name].timeout = item.timeout ftp_conn_params[item.name].port = item.port ftp_conn_params[item.name].dircache = item.dircache self.ftp = FTPFacade(ftp_conn_params) self.worker_config = Bunch() # Repo location so that AMQP subprocesses know how where to read # the server's configuration from. self.worker_config.repo_location = self.repo_location # The broker client for each of the worker threads. self.worker_config.broker_config = Bunch() self.worker_config.broker_config.name = 'worker-thread' self.worker_config.broker_config.broker_token = self.broker_token self.worker_config.broker_config.zmq_context = self.zmq_context self.worker_config.broker_config.broker_push_client_pull = self.broker_push_worker_pull self.worker_config.broker_config.client_push_broker_pull = self.worker_push_broker_pull self.worker_config.broker_config.broker_pub_client_sub = self.broker_pub_worker_sub # HTTP Basic Auth ba_config = Bunch() for item in self.odb.get_basic_auth_list(server.cluster.id): ba_config[item.name] = Bunch() ba_config[item.name].is_active = item.is_active ba_config[item.name].username = item.username ba_config[item.name].domain = item.domain ba_config[item.name].password = item.password # Technical accounts ta_config = Bunch() for item in self.odb.get_tech_acc_list(server.cluster.id): ta_config[item.name] = Bunch() ta_config[item.name].is_active = item.is_active ta_config[item.name].name = item.name ta_config[item.name].password = item.password ta_config[item.name].salt = item.salt wss_config = Bunch() for item in self.odb.get_wss_list(server.cluster.id): wss_config[item.name] = Bunch() wss_config[item.name].is_active = item.is_active wss_config[item.name].username = item.username wss_config[item.name].password = item.password wss_config[item.name].password_type = item.password_type wss_config[item.name].reject_empty_nonce_ts = item.reject_empty_nonce_ts wss_config[item.name].reject_stale_username = item.reject_stale_username wss_config[item.name].expiry_limit = item.expiry_limit wss_config[item.name].nonce_freshness = item.nonce_freshness # Security configuration of HTTP URLs. url_sec = self.odb.get_url_security(server) # All the HTTP/SOAP channels. http_soap = MultiDict() for item in self.odb.get_http_soap_list(server.cluster.id, 'channel'): _info = Bunch() _info[item.soap_action] = Bunch() _info[item.soap_action].id = item.id _info[item.soap_action].name = item.name _info[item.soap_action].is_internal = item.is_internal _info[item.soap_action].url_path = item.url_path _info[item.soap_action].method = item.method _info[item.soap_action].soap_version = item.soap_version _info[item.soap_action].service_id = item.service_id _info[item.soap_action].service_name = item.service_name _info[item.soap_action].impl_name = item.impl_name http_soap.add(item.url_path, _info) self.worker_config.basic_auth = ba_config self.worker_config.tech_acc = ta_config self.worker_config.wss = wss_config self.worker_config.url_sec = url_sec self.worker_config.http_soap = http_soap # The parallel server's broker client. The client's used to notify # all the server's AMQP subprocesses that they need to shut down. self.broker_client = BrokerClient() self.broker_client.name = 'parallel' self.broker_client.token = server.cluster.broker_token self.broker_client.zmq_context = self.zmq_context self.broker_client.client_push_broker_pull = self.parallel_push_broker_pull self.broker_client.init() self.broker_client.start()
def parse_formvars(environ, include_get_vars=True): """Parses the request, returning a MultiDict of form variables. If ``include_get_vars`` is true then GET (query string) variables will also be folded into the MultiDict. All values should be strings, except for file uploads which are left as ``FieldStorage`` instances. If the request was not a normal form request (e.g., a POST with an XML body) then ``environ['wsgi.input']`` won't be read. """ source = environ['wsgi.input'] if 'paste.parsed_formvars' in environ: parsed, check_source = environ['paste.parsed_formvars'] if check_source == source: if include_get_vars: parsed.update(parse_querystring(environ)) return parsed # @@: Shouldn't bother FieldStorage parsing during GET/HEAD and # fake_out_cgi requests type = environ.get('CONTENT_TYPE', '').lower() if ';' in type: type = type.split(';', 1)[0] fake_out_cgi = type not in ('', 'application/x-www-form-urlencoded', 'multipart/form-data') # FieldStorage assumes a default CONTENT_LENGTH of -1, but a # default of 0 is better: if not environ.get('CONTENT_LENGTH'): environ['CONTENT_LENGTH'] = '0' # Prevent FieldStorage from parsing QUERY_STRING during GET/HEAD # requests old_query_string = environ.get('QUERY_STRING','') environ['QUERY_STRING'] = '' if fake_out_cgi: input = StringIO('') old_content_type = environ.get('CONTENT_TYPE') old_content_length = environ.get('CONTENT_LENGTH') environ['CONTENT_LENGTH'] = '0' environ['CONTENT_TYPE'] = '' else: input = environ['wsgi.input'] fs = cgi.FieldStorage(fp=input, environ=environ, keep_blank_values=1) environ['QUERY_STRING'] = old_query_string if fake_out_cgi: environ['CONTENT_TYPE'] = old_content_type environ['CONTENT_LENGTH'] = old_content_length formvars = MultiDict() if isinstance(fs.value, list): for name in fs.keys(): values = fs[name] if not isinstance(values, list): values = [values] for value in values: if not value.filename: value = value.value formvars.add(name, value) environ['paste.parsed_formvars'] = (formvars, source) if include_get_vars: formvars.update(parse_querystring(environ)) return formvars
class QueryParser(object): """ The query parser will take any incoming query specifications and turn them into field-specific and general query parts. """ def __init__(self, query, terms, fields): self._query = query self._terms = terms self._fields = MultiDict(fields) @property def query(self): if not hasattr(self, '_combined_query'): parts = [self._query if self._query is not None else ''] for term in self._terms: if term.find(u' ') != -1: term = u"\"%s\"" % term parts.append(term.strip()) for field, value in self._fields.items(): if value.find(' ') != -1: value = u"\"%s\"" % value parts.append(u"%s:%s" % (field.strip(), value.strip())) self._combined_query = u' '.join(parts) return self._combined_query def _query_tokens(self): """ Split the query string, leaving quoted strings intact. """ if self._query: inside_quote = False buf = u'' for ch in self._query: if ch == u' ' and not inside_quote: if len(buf): yield buf.strip() buf = u'' elif ch == inside_quote: inside_quote = False elif ch in [u"\"", u"'"]: inside_quote = ch else: buf += ch if len(buf): yield buf.strip() def _parse_query(self): """ Decompose the query string into fields and terms. """ self._combined_fields = MultiDict(self._fields) self._combined_terms = list(self._terms) for token in self._query_tokens(): colon_pos = token.find(u':') if colon_pos != -1: field = token[:colon_pos] value = token[colon_pos+1:] value = value.strip('"').strip("'").strip() self._combined_fields.add(field, value) else: self._combined_terms.append(token) @property def fields(self): if not hasattr(self, '_combined_fields'): self._parse_query() return self._combined_fields @property def terms(self): if not hasattr(self, '_combined_terms'): self._parse_query() return self._combined_terms def validate(self): """ Check that this is a valid query. """ if not len(self.query): raise SearchError("No query has been specified") def __str__(self): return self.query def __repr__(self): return "Query(%s)" % self
def __init__(self, query, terms, fields): self._query = query self._terms = terms self._fields = MultiDict(fields)
def test_dict(): d = MultiDict({'a': 1}) assert d.items() == [('a', 1)] d['b'] = 2 d['c'] = 3 assert d.items() == [('a', 1), ('b', 2), ('c', 3)] d['b'] = 4 assert d.items() == [('a', 1), ('c', 3), ('b', 4)] d.add('b', 5) raises(KeyError, 'd.getone("b")') assert d.getall('b') == [4, 5] assert d.items() == [('a', 1), ('c', 3), ('b', 4), ('b', 5)] del d['b'] assert d.items() == [('a', 1), ('c', 3)] assert d.pop('xxx', 5) == 5 assert d.getone('a') == 1 assert d.popitem() == ('c', 3) assert d.items() == [('a', 1)] item = [] assert d.setdefault('z', item) is item assert d.items() == [('a', 1), ('z', item)] assert d.setdefault('y', 6) == 6 assert d.mixed() == {'a': 1, 'y': 6, 'z': item} assert d.dict_of_lists() == {'a': [1], 'y': [6], 'z': [item]} assert 'a' in d dcopy = d.copy() assert dcopy is not d assert dcopy == d d['x'] = 'x test' assert dcopy != d d[(1, None)] = (None, 1) assert d.items() == [('a', 1), ('z', []), ('y', 6), ('x', 'x test'), ((1, None), (None, 1))]
def get_url_security(self, cluster_id, connection=None): """ Returns the security configuration of HTTP URLs. """ # What DB class to fetch depending on the string value of the security type. sec_type_db_class = { 'tech_acc': TechnicalAccount, 'basic_auth': HTTPBasicAuth, 'wss': WSSDefinition } result = MultiDict() query = http_soap_security_list(self._session, cluster_id, connection) columns = Bunch() # So ConfigDict has its data in the format it expects for c in query.statement.columns: columns[c.name] = None for item in query.all(): _info = Bunch() _info[item.soap_action] = Bunch() _info[item.soap_action].is_active = item.is_active _info[item.soap_action].transport = item.transport _info[item.soap_action].data_format = item.data_format if item.security_id: _info[item.soap_action].sec_def = Bunch() # Will raise KeyError if the DB gets somehow misconfigured. db_class = sec_type_db_class[item.sec_type] sec_def = self._session.query(db_class).\ filter(db_class.id==item.security_id).\ one() # Common things first _info[item.soap_action].sec_def.name = sec_def.name _info[item.soap_action].sec_def.password = sec_def.password _info[item.soap_action].sec_def.sec_type = item.sec_type if item.sec_type == security_def_type.tech_account: _info[item.soap_action].sec_def.salt = sec_def.salt elif item.sec_type == security_def_type.basic_auth: _info[item.soap_action].sec_def.username = sec_def.username _info[item.soap_action].sec_def.password = sec_def.password _info[item.soap_action].sec_def.realm = sec_def.realm elif item.sec_type == security_def_type.wss: _info[item.soap_action].sec_def.username = sec_def.username _info[item.soap_action].sec_def.password = sec_def.password _info[ item. soap_action].sec_def.password_type = sec_def.password_type _info[ item. soap_action].sec_def.reject_empty_nonce_creat = sec_def.reject_empty_nonce_creat _info[ item. soap_action].sec_def.reject_stale_tokens = sec_def.reject_stale_tokens _info[ item. soap_action].sec_def.reject_expiry_limit = sec_def.reject_expiry_limit _info[ item. soap_action].sec_def.nonce_freshness_time = sec_def.nonce_freshness_time else: _info[item.soap_action].sec_def = ZATO_NONE result.add(item.url_path, _info) return result, columns
def _test_unicode_dict(decode_param_names=False): d = UnicodeMultiDict(MultiDict({'a': 'a test'})) d.encoding = 'utf-8' d.errors = 'ignore' if decode_param_names: key_str = unicode d.decode_keys = True else: key_str = str def assert_unicode(obj): assert isinstance(obj, unicode) def assert_key_str(obj): assert isinstance(obj, key_str) def assert_unicode_item(obj): key, value = obj assert isinstance(key, key_str) assert isinstance(value, unicode) assert d.items() == [('a', u'a test')] map(assert_key_str, d.keys()) map(assert_unicode, d.values()) d['b'] = '2 test' d['c'] = '3 test' assert d.items() == [('a', u'a test'), ('b', u'2 test'), ('c', u'3 test')] map(assert_unicode_item, d.items()) d['b'] = '4 test' assert d.items() == [('a', u'a test'), ('c', u'3 test'), ('b', u'4 test')] map(assert_unicode_item, d.items()) d.add('b', '5 test') raises(KeyError, 'd.getone("b")') assert d.getall('b') == [u'4 test', u'5 test'] map(assert_unicode, d.getall('b')) assert d.items() == [('a', u'a test'), ('c', u'3 test'), ('b', u'4 test'), ('b', u'5 test')] map(assert_unicode_item, d.items()) del d['b'] assert d.items() == [('a', u'a test'), ('c', u'3 test')] map(assert_unicode_item, d.items()) assert d.pop('xxx', u'5 test') == u'5 test' assert isinstance(d.pop('xxx', u'5 test'), unicode) assert d.getone('a') == u'a test' assert isinstance(d.getone('a'), unicode) assert d.popitem() == ('c', u'3 test') d['c'] = '3 test' assert_unicode_item(d.popitem()) assert d.items() == [('a', u'a test')] map(assert_unicode_item, d.items()) item = [] assert d.setdefault('z', item) is item items = d.items() assert items == [('a', u'a test'), ('z', item)] assert isinstance(items[1][0], key_str) assert isinstance(items[1][1], list) assert isinstance(d.setdefault('y', 'y test'), unicode) assert isinstance(d['y'], unicode) assert d.mixed() == {u'a': u'a test', u'y': u'y test', u'z': item} assert d.dict_of_lists() == { u'a': [u'a test'], u'y': [u'y test'], u'z': [item] } del d['z'] map(assert_unicode_item, d.mixed().iteritems()) map(assert_unicode_item, [(k, v[0]) for \ k, v in d.dict_of_lists().iteritems()]) assert u'a' in d dcopy = d.copy() assert dcopy is not d assert dcopy == d d['x'] = 'x test' assert dcopy != d d[(1, None)] = (None, 1) assert d.items() == [('a', u'a test'), ('y', u'y test'), ('x', u'x test'), ((1, None), (None, 1))] item = d.items()[-1] assert isinstance(item[0], tuple) assert isinstance(item[1], tuple) fs = cgi.FieldStorage() fs.name = 'thefile' fs.filename = 'hello.txt' fs.file = StringIO('hello') d['f'] = fs ufs = d['f'] assert isinstance(ufs, cgi.FieldStorage) assert ufs is not fs assert ufs.name == fs.name assert isinstance(ufs.name, key_str) assert ufs.filename == fs.filename assert isinstance(ufs.filename, unicode) assert isinstance(ufs.value, str) assert ufs.value == 'hello'
class GoalAnalyser(object): """Takes a log file and some filters for URL specific stats and generates CSV files with the result of the DailyStatistics filter""" def __init__(self, log, location, urls=None, avgs=[1, 7], past_only=False, config=None): super(GoalAnalyser, self).__init__() self.config = config self.log = log if urls is None: if self.config is None: self.urls = {} else: self.urls = config.urls() else: self.urls = urls self.avgs = avgs self.dir = location self.past_only = past_only self.statscounters = MultiDict() self.outputs = {} self.files = {} self.existing_dates = {} self.parse = logparser() self.log_entries = {} if self.config is not None: for section in self.config.sections(): log = dict(self.config.items(section)).get('log', '').lower() if log in ('true', 'yes', 'on'): fn = '%s.log' % section self.log_entries[section] = fn self.files[fn] = open(os.path.join(self.dir, fn), 'w') def _instantiateFilters(self): for name in self.urls: self.statscounters[name] = DailyStatistics(self.avgs) def _instantiateCSVWriters(self): keys = ['date', ] + sorted(DailyStatistics(self.avgs).stats().keys()) for name in self.urls: location = os.path.join(self.dir, '%s_stats.csv' % name) if os.path.exists(location): # We are going to add to an existing file. backing = open(location, 'r+') reader = DictReader(backing) self.existing_dates[name] = [r['date'] for r in reader] else: backing = open(location, 'w') self.files[name] = backing writer = DictWriter(backing, keys) if self.existing_dates.get(name, None) is None: writer.writerow(dict(zip(keys, keys))) self.outputs[name] = writer def filterForLine(self, line): """Take a parsed log line and return the rule name that it matches or None if none match.""" status = int(line.get('status', '200')) if status >= 400 and status < 600: return method = line['method'] url = line['url'] qs = line['querystring'] if qs is not None: url += qs for name in self.statscounters.keys(): method_name, url_pattern = self.urls[name] if method_name != method: continue if url_pattern.match(url): return name warnings.warn("%s for %s is not classified" % (method, url)) def __call__(self): self._instantiateFilters() self._instantiateCSVWriters() # We know the dates are in order, so parse them and groupby their date iterable = itertools.imap(self.parse, self.log) iterable = itertools.ifilter(lambda x: x is not None, iterable) days = itertools.groupby(iterable, getDateForLine) existing = self.existing_dates # if avgs goes back several days we have to gather statistics # for days in existing today = datetime.date.today() statsdays = set([str(today-datetime.timedelta(days=x)) for x in range(int(self.past_only),max(self.avgs)+int(self.past_only))]) for day, iterable in days: if self.past_only and day == today: continue # Duplicate the iterator for each day, find the responsible rule # name and turn it into a dictionary.iteritems() style iterator. parsed, destination = itertools.tee(iterable) destination = itertools.imap(self.filterForLine, destination) classified = itertools.izip(destination, parsed) for destination, entry in classified: # Pass the line onto the underlying stats class if day.isoformat() not in statsdays and day.isoformat() in existing.get(destination, []): continue if destination is None: continue self.statscounters[destination].process(entry, day) fn = self.log_entries.get(destination) if fn is not None: entry_info = dict(entry) entry_info['querystring'] = entry_info['querystring'] or '' self.files[fn].write(LOG_FORMAT.format(**entry_info)) for name in self.urls: # Don't duplicate dates in csv file if day.isoformat() in set(sum(existing.values(), [])): continue stats = self.statscounters[name].stats() if not any(stats.values()): # We have no data at all, skip this warnings.warn("No data for %s on %s" % (name, day.isoformat())) continue stats['date'] = day.isoformat() self.outputs[name].writerow(stats) self.finish() def finish(self): for f in self.files.values(): f.flush() os.fsync(f.fileno()) f.close()
def parse_formvars(environ, include_get_vars=True): """Parses the request, returning a MultiDict of form variables. If ``include_get_vars`` is true then GET (query string) variables will also be folded into the MultiDict. All values should be strings, except for file uploads which are left as ``FieldStorage`` instances. If the request was not a normal form request (e.g., a POST with an XML body) then ``environ['wsgi.input']`` won't be read. """ source = environ['wsgi.input'] if 'paste.parsed_formvars' in environ: parsed, check_source = environ['paste.parsed_formvars'] if check_source == source: if include_get_vars: parsed.update(parse_querystring(environ)) return parsed # @@: Shouldn't bother FieldStorage parsing during GET/HEAD and # fake_out_cgi requests type = environ.get('CONTENT_TYPE', '').lower() if ';' in type: type = type.split(';', 1)[0] fake_out_cgi = type not in ('', 'application/x-www-form-urlencoded', 'multipart/form-data') # FieldStorage assumes a default CONTENT_LENGTH of -1, but a # default of 0 is better: if not environ.get('CONTENT_LENGTH'): environ['CONTENT_LENGTH'] = '0' # Prevent FieldStorage from parsing QUERY_STRING during GET/HEAD # requests old_query_string = environ.get('QUERY_STRING','') environ['QUERY_STRING'] = '' if fake_out_cgi: input = six.BytesIO(b'') old_content_type = environ.get('CONTENT_TYPE') old_content_length = environ.get('CONTENT_LENGTH') environ['CONTENT_LENGTH'] = '0' environ['CONTENT_TYPE'] = '' else: input = environ['wsgi.input'] fs = cgi.FieldStorage(fp=input, environ=environ, keep_blank_values=1) environ['QUERY_STRING'] = old_query_string if fake_out_cgi: environ['CONTENT_TYPE'] = old_content_type environ['CONTENT_LENGTH'] = old_content_length formvars = MultiDict() if isinstance(fs.value, list): for name in fs.keys(): values = fs[name] if not isinstance(values, list): values = [values] for value in values: if not value.filename: value = value.value formvars.add(name, value) environ['paste.parsed_formvars'] = (formvars, source) if include_get_vars: formvars.update(parse_querystring(environ)) return formvars
def __init__(self, items=()): self.v = MultiDict([(normalize(key), val) for (key, val) in items]) self.changed = False
try: since_time = h.date_str_to_datetime(since_time_str) except ValueError, inst: return self._finish_bad_request('ValueError: %s' % inst) else: return self._finish_bad_request( _("Missing search term ('since_id=UUID' or " + " 'since_time=TIMESTAMP')")) revs = model.Session.query(model.Revision) \ .filter(model.Revision.timestamp > since_time) \ .order_by(model.Revision.timestamp) \ .limit(50) # reasonable enough for a page return self._finish_ok([rev.id for rev in revs]) elif register in ['dataset', 'package', 'resource']: try: params = MultiDict(self._get_search_params(request.params)) except ValueError, e: return self._finish_bad_request( _('Could not read parameters: %r' % e)) # if using API v2, default to returning the package ID if # no field list is specified if register in ['dataset', 'package'] and not params.get('fl'): params['fl'] = 'id' if ver == 2 else 'name' try: if register == 'resource': query = search.query_for(model.Resource) # resource search still uses ckan query parser options = search.QueryOptions()
class MimeHeaders(object): """Dictionary-like object that preserves the order and supports multiple values for the same key, knows whether it has been changed after the creation """ def __init__(self, items=()): self.v = MultiDict([(normalize(key), val) for (key, val) in items]) self.changed = False def __getitem__(self, key): return self.v.get(normalize(key), None) def __len__(self): return len(self.v) def __iter__(self): return iter(self.v) def __contains__(self, key): return normalize(key) in self.v def __setitem__(self, key, value): self.v[normalize(key)] = _remove_newlines(value) self.changed = True def __delitem__(self, key): del self.v[normalize(key)] self.changed = True def __nonzero__(self): return len(self.v) > 0 def prepend(self, key, val): self.v._items.insert(0, (key, _remove_newlines(val))) self.changed = True def add(self, key, value): """Adds header without changing the existing headers with same name""" self.v.add(normalize(key), _remove_newlines(value)) self.changed = True def keys(self): """ Returns the keys. (message header names) It remembers the order in which they were added, what is really important """ return self.v.keys() def transform(self, fn): """Accepts a function, getting a key, val and returning a new pair of key, val and applies the function to all header, value pairs in the message. """ changed = [False] def tracking_fn(key, val): new_key, new_val = fn(key, val) if new_val != val or new_key != key: changed[0] = True return new_key, new_val v = MultiDict(tracking_fn(key, val) for key, val in self.v.iteritems()) if changed[0]: self.v = v self.changed = True def items(self): """ Returns header,val pairs in the preserved order. """ return list(self.iteritems()) def iteritems(self): """ Returns iterator header,val pairs in the preserved order. """ return self.v.iteritems() def get(self, key, default=None): """ Returns header value (case-insensitive). """ return self.v.get(normalize(key), default) def getall(self, key): """ Returns all header values by the given header name (case-insensitive) """ return self.v.getall(normalize(key)) def have_changed(self): """Tells whether someone has altered the headers after creation""" return self.changed def __str__(self): return str(self.v) @classmethod def from_stream(cls, stream): """Takes a stream and reads the headers, decodes headers to unicode dict like object""" return cls(parse_stream(stream)) def to_stream(self, stream): """Takes a stream and serializes headers in a mime format""" for h, v in self.v.iteritems(): try: h = h.encode('ascii') except UnicodeDecodeError: raise EncodingError("Non-ascii header name") stream.write("{0}: {1}\r\n".format(h, to_mime(h, v)))
def __init__(self, items=()): self.v = MultiDict( [(normalize(key), val) for (key, val) in items]) self.changed = False
def search(self, ver=None, register=None): log.debug('search %s params: %r', register, request.params) if register == 'revision': since_time = None if 'since_id' in request.params: id = request.params['since_id'] if not id: return self._finish_bad_request( _(u'No revision specified')) rev = model.Session.query(model.Revision).get(id) if rev is None: return self._finish_not_found( _(u'There is no revision with id: %s') % id) since_time = rev.timestamp elif 'since_time' in request.params: since_time_str = request.params['since_time'] try: since_time = h.date_str_to_datetime(since_time_str) except ValueError as inst: return self._finish_bad_request('ValueError: %s' % inst) else: return self._finish_bad_request( _("Missing search term ('since_id=UUID' or " + " 'since_time=TIMESTAMP')")) revs = model.Session.query(model.Revision) \ .filter(model.Revision.timestamp > since_time) \ .order_by(model.Revision.timestamp) \ .limit(50) # reasonable enough for a page return self._finish_ok([rev.id for rev in revs]) elif register in ['dataset', 'package', 'resource']: try: params = MultiDict(self._get_search_params(request.params)) except ValueError as e: return self._finish_bad_request( _('Could not read parameters: %r' % e)) # if using API v2, default to returning the package ID if # no field list is specified if register in ['dataset', 'package'] and not params.get('fl'): params['fl'] = 'id' if ver == 2 else 'name' try: if register == 'resource': query = search.query_for(model.Resource) # resource search still uses ckan query parser options = search.QueryOptions() for k, v in params.items(): if (k in search.DEFAULT_OPTIONS.keys()): options[k] = v options.update(params) options.username = c.user options.search_tags = False options.return_objects = False query_fields = MultiDict() for field, value in params.items(): field = field.strip() if field in search.DEFAULT_OPTIONS.keys() or \ field in IGNORE_FIELDS: continue values = [value] if isinstance(value, list): values = value for v in values: query_fields.add(field, v) results = query.run(query=params.get('q'), fields=query_fields, options=options) else: # For package searches in API v3 and higher, we can pass # parameters straight to Solr. if ver in [1, 2]: # Otherwise, put all unrecognised ones into the q # parameter params = search.\ convert_legacy_parameters_to_solr(params) query = search.query_for(model.Package) # Remove any existing fq param and set the capacity to # public if 'fq' in params: del params['fq'] params['fq'] = '+capacity:public' # if callback is specified we do not want to send that to # the search if 'callback' in params: del params['callback'] results = query.run(params) return self._finish_ok(results) except search.SearchError as e: log.exception(e) return self._finish_bad_request(_('Bad search option: %s') % e) else: return self._finish_not_found(_('Unknown register: %s') % register)
class SubscribeTests(BaseTest): default_data = MultiDict({ 'hub.verify': 'sync', 'hub.callback': 'http://httpbin.org/get', 'hub.mode': "subscribe", 'hub.topic': "http://www.google.com/" }) challenge = "abcdefg" def test_subscribe(self): request = Request.blank('/subscribe') info = subscribe(None, request) self.assertEqual(info.status_code, 405) def test_invalid_content_type(self): headers = [("Content-Type", "text/plain")] request = self.r( '/subscribe', headers=headers, POST={"thing": "thing"} ) info = subscribe(None, request) self.assertEqual(info.status_code, 406) self.assertEqual( info.headers['Accept'], 'application/x-www-form-urlencoded' ) def test_invalid_verify_type(self): data = {"hub.verify": "bogus"} request = self.r( '/subscribe', POST=data ) info = subscribe(None, request) self.assertEqual(info.status_code, 400) self.assertEqual(info.headers['Content-Type'], 'text/plain') self.assertTrue("hub.verify" in info.body) @patch.object(Hub, 'get_challenge_string') def test_multiple_verify_types_one_valid(self, mock_get_challenge_string): data = self.default_data.copy() mock_get_challenge_string.return_value = self.challenge del data["hub.verify"] data.add('hub.verify', 'bogus') data.add('hub.verify', 'sync') request = self.r( '/subscribe', POST=data ) with patch('requests.get', new_callable=MockResponse, content=self.challenge, status_code=200): info = subscribe(None, request) self.assertEqual(info.status_code, 204) def test_multiple_invalid_verify_types(self): data = self.default_data.copy() del data["hub.verify"] data.add('hub.verify', 'bad') data.add('hub.verify', 'wrong') request = self.r( '/subscribe', POST=data ) info = subscribe(None, request) self.assertEqual(info.status_code, 400) self.assertEqual(info.headers['Content-Type'], 'text/plain') self.assertTrue("hub.verify" in info.body) def test_invalid_callback(self): data = self.default_data.copy() del data['hub.callback'] data.add("hub.callback", "www.google.com") request = self.r( '/subscribe', POST=data ) info = subscribe(None, request) self.assertEqual(info.status_code, 400) self.assertTrue('hub.callback' in info.body) @patch.object(Hub, 'get_challenge_string') def test_valid_callback(self, mock_get_challenge_string): data = self.default_data.copy() mock_get_challenge_string.return_value = self.challenge request = self.r( '/subscribe', POST=data ) with patch('requests.get', new_callable=MockResponse, content=self.challenge, status_code=200): info = subscribe(None, request) self.assertEqual(info.status_code, 204) def test_invalid_mode(self): data = self.default_data.copy() del data['hub.mode'] data.add('hub.mode', 'bad') request = self.r( '/subscribe', POST=data, ) info = subscribe(None, request) self.assertEqual(info.status_code, 400) self.assertTrue('hub.mode' in info.body) @patch.object(Hub, 'get_challenge_string') def test_valid_mode(self, mock_get_challenge_string): data = self.default_data.copy() mock_get_challenge_string.return_value = self.challenge request = self.r( '/subscribe', POST=data ) with patch('requests.get', new_callable=MockResponse, content=self.challenge, status_code=200): info = subscribe(None, request) self.assertEqual(info.status_code, 204) @patch.object(Hub, 'get_challenge_string') def test_valid_topic(self, mock_get_challenge_string): data = self.default_data.copy() mock_get_challenge_string.return_value = self.challenge request = self.r( '/subscribe', POST=data ) with patch('requests.get', new_callable=MockResponse, content=self.challenge, status_code=200): info = subscribe(None, request) self.assertEqual(info.status_code, 204) def test_invalid_topic(self): data = self.default_data.copy() del data['hub.topic'] data.add('hub.topic', 'http://google.com/#fragment') request = self.r( '/subscribe', POST=data ) info = subscribe(None, request) self.assertEqual(info.status_code, 400) self.assertTrue('hub.topic' in info.body) def test_not_verified_subscription(self): data = self.default_data.copy() del data["hub.callback"] data.add('hub.callback', 'http://httpbin.org/status/404') request = self.r( '/subscribe', POST=data ) with patch('requests.get', new_callable=MockResponse, status_code=404): info = subscribe(None, request) self.assertEqual(info.status_code, 409) @patch.object(Hub, 'get_challenge_string') def test_subscribe_to_actual_topic(self, mock_get_challenge_string): """ Make sure that the topic subscribed to is same as published. """ data = self.default_data.copy() mock_get_challenge_string.return_value = self.challenge request = self.r( '/subscribe', POST=data ) # Publish the URL first. self.root.publish('http://www.google.com/') urls = { 'http://www.google.com/': MockResponse(content=good_atom), 'http://httpbin.org/get': MockResponse( content=self.challenge, status_code=200 ) } with patch('requests.get', new_callable=MultiResponse, mapping=urls): subscribe(None, request) hub = self.root topic = hub.topics.get('http://www.google.com/') subscriber = hub.subscribers.get('http://httpbin.org/get') self.assertEqual( topic, subscriber.topics.get('http://www.google.com/'))
return thisvalue i = get_indexpage(indexpage) for inturl in i: idsm = re.search('TDATA\-(\S+)', inturl, re.MULTILINE) xmldatafile = 'somefile' if idsm: xmldatafile = "./data/%s.xml" % idsm.group(1) xmlfile = open(xmldatafile,'w') html = read_remote_page(inturl) parsed_html = BeautifulSoup(html) #, "html.parser") items = parsed_html.findAll('p') #, attrs={'class':'toolDataItemName'}) mainkey = '' d = MultiDict() DEBUG = 0 for item in items: cleanitem = item thiskey = item.find('b') #, attrs={'img':'class bdtIcon'}) if thiskey: value = str(item) value = value.replace(str(thiskey), '') value = cleaner(value) if DEBUG: print "X %s" % value mainkey = thiskey if value: d.add(thiskey.text, value) else: if mainkey:
elif register == 'package' or register == 'resource': try: params = self._get_search_params(request.params) except ValueError, e: return self._finish_bad_request( gettext('Could not read parameters: %r' % e)) options = QueryOptions() for k, v in params.items(): if (k in DEFAULT_OPTIONS.keys()): options[k] = v options.update(params) options.username = c.user options.search_tags = False options.return_objects = False query_fields = MultiDict() for field, value in params.items(): field = field.strip() if field in DEFAULT_OPTIONS.keys() or \ field in IGNORE_FIELDS: continue values = [value] if isinstance(value, list): values = value for v in values: query_fields.add(field, v) if register == 'package': options.ref_entity_with_attr = 'id' if ver == '2' else 'name' try: backend = None
def test_dict(): d = MultiDict({'a': 1}) assert list(d.items()) == [('a', 1)] d['b'] = 2 d['c'] = 3 assert list(d.items()) == [('a', 1), ('b', 2), ('c', 3)] d['b'] = 4 assert list(d.items()) == [('a', 1), ('c', 3), ('b', 4)] d.add('b', 5) assert_raises(KeyError, d.getone, "b") assert d.getall('b') == [4, 5] assert list(d.items()) == [('a', 1), ('c', 3), ('b', 4), ('b', 5)] del d['b'] assert list(d.items()) == [('a', 1), ('c', 3)] assert d.pop('xxx', 5) == 5 assert d.getone('a') == 1 assert d.popitem() == ('c', 3) assert list(d.items()) == [('a', 1)] item = [] assert d.setdefault('z', item) is item assert list(d.items()) == [('a', 1), ('z', item)] assert d.setdefault('y', 6) == 6 assert d.mixed() == {'a': 1, 'y': 6, 'z': item} assert d.dict_of_lists() == {'a': [1], 'y': [6], 'z': [item]} assert 'a' in d dcopy = d.copy() assert dcopy is not d assert dcopy == d d['x'] = 'x test' assert dcopy != d d[(1, None)] = (None, 1) assert list(d.items()) == [('a', 1), ('z', []), ('y', 6), ('x', 'x test'), ((1, None), (None, 1))]