def getChild(self, path, request): """See twisted.web.Resource.getChild. """ self.restat() if not self.isdir(): return self.childNotFound if path: fpath = self.child(path) else: fpath = self.childSearchPreauth(*self.indexNames) if fpath is None: return self.directoryListing() if not fpath.exists(): fpath = fpath.siblingExtensionSearch(*self.ignoredExts) if fpath is None: return self.childNotFound if platformType == "win32": # don't want .RPY to be different than .rpy, since that would allow # source disclosure. processor = InsensitiveDict(self.processors).get( fpath.splitext()[1]) else: processor = self.processors.get(fpath.splitext()[1]) if processor: return resource.IResource(processor(fpath.path, self.registry)) return self.createSimilarFile(fpath.path)
def setIndexLink(template, indexFilename): """ Insert a link to an index document. Any node with a C{class} attribute set to C{index-link} will have its tag name changed to C{a} and its C{href} attribute set to C{indexFilename}. @type template: A DOM Node or Document @param template: The output template which defines the presentation of the version information. @type indexFilename: C{str} @param indexFilename: The address of the index document to which to link. If any C{False} value, this function will remove all index-link nodes. @return: C{None} """ indexLinks = domhelpers.findElementsWithAttribute(template, "class", "index-link") for link in indexLinks: if indexFilename is None: link.parentNode.removeChild(link) else: link.nodeName = link.tagName = link.endTagName = 'a' link.attributes = InsensitiveDict({'href': indexFilename})
def getChild(self, path, request): """ If this L{File}"s path refers to a directory, return a L{File} referring to the file named C{path} in that directory. If C{path} is the empty string, return a L{DirectoryLister} instead. @param path: The current path segment. @type path: L{bytes} @param request: The incoming request. @type request: An that provides L{twisted.web.iweb.IRequest}. @return: A resource representing the requested file or directory, or L{NoResource} if the path cannot be accessed. @rtype: An object that provides L{resource.IResource}. """ if isinstance(path, bytes): try: # Request calls urllib.unquote on each path segment, # leaving us with raw bytes. path = path.decode("utf-8") except UnicodeDecodeError: log.err( None, "Could not decode path segment as utf-8: %r" % (path, )) return self.childNotFound self.restat(reraise=False) if not self.isdir(): return self.childNotFound if path: try: fpath = self.child(path) except filepath.InsecurePath: return self.childNotFound else: fpath = self.childSearchPreauth(*self.indexNames) if fpath is None: return self.directoryListing() if not fpath.exists(): fpath = fpath.siblingExtensionSearch(*self.ignoredExts) if fpath is None: return self.childNotFound extension = fpath.splitext()[1] if platformType == "win32": # don't want .RPY to be different than .rpy, since that would allow # source disclosure. processor = InsensitiveDict(self.processors).get(extension) else: processor = self.processors.get(extension) if processor: return resource.IResource(processor(fpath.path, self.registry)) return self.createSimilarFile(fpath.path)
def __init__(self, tagName, attributes=None, parentNode=None, filename=None, markpos=None, case_insensitive=1, namespace=None): Node.__init__(self, parentNode) preserve_case = not case_insensitive tagName = tagName if preserve_case else tagName.lower() unescaped = unescape_dict(attributes or {}) if case_insensitive: self.attributes = InsensitiveDict(unescaped, preserve=preserve_case) else: self.attributes = unescaped self.preserve_case = not case_insensitive self.case_insensitive = case_insensitive self.endTagName = self.nodeName = self.tagName = tagName self._filename = filename self._markpos = markpos self.namespace = namespace self.tag_is_blockelement = tagName in self.BLOCKELEMENTS self.tag_is_nice_format = tagName in self.NICEFORMATS self.tag_is_singleton = tagName.lower() in self.SINGLETONS
def __init__(self, tagName, attributes=None, parentNode=None, filename=None, markpos=None, caseInsensitive=1, preserveCase=0, namespace=None): Node.__init__(self, parentNode) self.preserveCase = preserveCase or not caseInsensitive self.caseInsensitive = caseInsensitive if not preserveCase: tagName = tagName.lower() if attributes is None: self.attributes = {} else: self.attributes = attributes for k, v in self.attributes.items(): self.attributes[k] = unescape(v) if caseInsensitive: self.attributes = InsensitiveDict(self.attributes, preserve=preserveCase) self.endTagName = self.nodeName = self.tagName = tagName self._filename = filename self._markpos = markpos self.namespace = namespace
def __init__(self, url, method='GET', postdata=None, headers=None, agent="Twisted PageGetter", timeout=0, cookies=None, followRedirect=True, redirectLimit=20): self.followRedirect = followRedirect self.redirectLimit = redirectLimit self._redirectCount = 0 self.timeout = timeout self.agent = agent if cookies is None: cookies = {} self.cookies = cookies if headers is not None: self.headers = InsensitiveDict(headers) else: self.headers = InsensitiveDict() if postdata is not None: self.headers.setdefault('Content-Length', len(postdata)) # just in case a broken http/1.1 decides to keep connection alive self.headers.setdefault("connection", "close") self.postdata = postdata self.method = method self.setURL(url) #print( 'method: {}\nurl:{}\nheaders:{}\npostdata:{}'.format( method, url, self.headers, postdata ) ) self.waiting = 1 # Fixes Twisted 11.1.0+ support as HTTPClientFactory is expected # to have _disconnectedDeferred. See Twisted r32329. # As Scrapy implements it's own logic to handle redirects is not # needed to add the callback _waitForDisconnect. # Specifically this avoids the AttributeError exception when # clientConnectionFailed method is called. self._disconnectedDeferred = defer.Deferred() self.deferred = defer.Deferred() self.response_headers = None
def __init__(self, url, method='GET', postdata=None, headers=None, agent="Twisted PageGetter", timeout=0, cookies=None, followRedirect=True, redirectLimit=20, afterFoundGet=False): self.followRedirect = followRedirect self.redirectLimit = redirectLimit self._redirectCount = 0 self.timeout = timeout self.agent = agent self.afterFoundGet = afterFoundGet if cookies is None: cookies = {} self.cookies = cookies if headers is not None: self.headers = InsensitiveDict(headers) else: self.headers = InsensitiveDict() if postdata is not None: self.headers.setdefault('Content-Length', len(postdata)) # just in case a broken http/1.1 decides to keep connection alive self.headers.setdefault("connection", "close") self.postdata = postdata self.method = method self.setURL(url) self.waiting = 1 self._disconnectedDeferred = defer.Deferred() self.deferred = defer.Deferred() # Make sure the first callback on the result Deferred pauses the # callback chain until the request connection is closed. self.deferred.addBoth(self._waitForDisconnect) self.response_headers = None
def test01_proxy(self): data = '{"type": "PacketPing"}' headers = InsensitiveDict() headers.setdefault('Content-Length', len(data)) headers.setdefault("connection", "close") headers.setdefault("proxy-connection", "foo") host = '127.0.0.1' port = 19481 path = '/POKER_REST' factory = pokerrestclient.PokerProxyClientFactory('POST', path, '1.1', headers, data, MockRequest(), 6, host + ':' + str(port) + path) reactor.connectTCP(host, port, factory) factory.deferred.addCallback(self.assertNotEquals, None) return factory.deferred
def __init__(self, host, port, path, data, timeout = 60): self.timeout = timeout self.agent = "RestClient" self.headers = InsensitiveDict() self.headers.setdefault('Content-Length', len(data)) self.headers.setdefault("connection", "close") self.method = 'POST' self.url = 'http://' + host + ':' + str(port) + path self.postdata = data self.host = host self.port = port self.path = path self.waiting = 1 self.deferred = defer.Deferred() self.response_headers = None self.cookies = {}
def __init__(self, factory, plugin): self.factory = factory self.nickname = plugin.nickname.encode('ascii') self.realname = plugin.realname.encode('ascii') self.username = plugin.ident.encode('ascii') self.ns_username = plugin.username self.ns_password = plugin.password self.password = plugin.server_password.encode('ascii') self.channel = plugin.channel.encode('ascii') self.console = plugin.console self.irc_message = plugin.irc_message self.irc_action = plugin.irc_action self.irc_chat_status = plugin.irc_chat_status self.mangle_username = plugin.mangle_username self.users = InsensitiveDict() self.cap_requests = set()
def __init__(self, headers): self.expires = None self.mtime = None self.length = 0 self.start = 0 self.size = 0 self.mimeType = None headers = InsensitiveDict(headers) encoding = headers.get("Transfer-Encoding", None) if encoding == 'chunked': raise errors.FlumotionError("Chunked transfer not supported") expires = headers.get("Expires", None) if expires is not None: try: self.expires = http.stringToDatetime(expires) except: self.expires = 0 lastmod = headers.get("Last-Modified", None) if lastmod is not None: self.mtime = http.stringToDatetime(lastmod) range = headers.get("Content-Range", None) length = headers.get("Content-Length", None) if range is not None: start, end, total = http.parseContentRange(range) self.start = start self.length = total if length is not None: self.size = int(length) else: self.size = end - start elif length is not None: self.length = int(length) self.size = int(length) else: raise errors.FlumotionError("Can't get length/size from headers", headers) ctype = headers.get("Content-Type", None) if ctype is not None: self.mimeType, _pdict = cgi.parse_header(ctype)
def locateChild(self, ctx, segments): r = self.children.get(segments[0], None) if r: return r, segments[1:] path = segments[0] self.fp.restat() if not self.fp.isdir(): return rend.NotFound if path: fpath = self.fp.child(path) else: fpath = self.fp.childSearchPreauth(*self.indexNames) if fpath is None: if self.allowListing: return self.directoryListing(), segments[1:] else: return rend.NotFound if not fpath.exists(): fpath = fpath.siblingExtensionSearch(*self.ignoredExts) if fpath is None: return rend.NotFound # Don't run processors on directories - if someone wants their own # customized directory rendering, subclass File instead. if fpath.isfile(): if platformType == "win32": # don't want .RPY to be different than .rpy, since that # would allow source disclosure. processor = InsensitiveDict(self.processors).get( fpath.splitext()[1]) else: processor = self.processors.get(fpath.splitext()[1]) if processor: return (inevow.IResource(processor(fpath.path, self.registry)), segments[1:]) return self.createSimilarFile(fpath.path), segments[1:]
def __init__(self, dn, attributes={}): """ Initialize the object. @param dn: Distinguished Name of the object, as a string. @param attributes: Attributes of the object. A dictionary of attribute types to list of attribute values. """ self._attributes = InsensitiveDict() self.dn = distinguishedname.DistinguishedName(dn) for k, vs in attributes.items(): if k not in self._attributes: self._attributes[k] = [] self._attributes[k].extend(vs) for k, vs in self._attributes.items(): self._attributes[k] = self.buildAttributeSet(k, vs)
def getChild(self, path, request): """ If this L{File}'s path refers to a directory, return a L{File} referring to the file named C{path} in that directory. If C{path} is the empty string, return a L{DirectoryLister} instead. """ self.restat(reraise=False) if not self.isdir(): return self.childNotFound if path: try: fpath = self.child(path) except filepath.InsecurePath: return self.childNotFound else: fpath = self.childSearchPreauth(*self.indexNames) if fpath is None: return self.directoryListing() if not fpath.exists(): fpath = fpath.siblingExtensionSearch(*self.ignoredExts) if fpath is None: return self.childNotFound if platformType == "win32": # don't want .RPY to be different than .rpy, since that would allow # source disclosure. processor = InsensitiveDict(self.processors).get( fpath.splitext()[1]) else: processor = self.processors.get(fpath.splitext()[1]) if processor: return resource.IResource(processor(fpath.path, self.registry)) return self.createSimilarFile(fpath.path)
def index(document, filename, chapterReference): """ Extract index entries from the given document and store them for later use and insert named anchors so that the index can link back to those entries. Any node with a C{class} attribute set to C{index} is considered an index entry. @type document: A DOM Node or Document @param document: The input document which contains all of the content to be presented. @type filename: C{str} @param filename: A link to the output for the given document which will be included in the index to link to any index entry found here. @type chapterReference: ??? @param chapterReference: ??? @return: C{None} """ entries = domhelpers.findElementsWithAttribute(document, "class", "index") if not entries: return i = 0 for entry in entries: i += 1 anchor = 'index%02d' % i if chapterReference: ref = getSectionReference(entry) or chapterReference else: ref = 'link' indexer.addEntry(filename, anchor, entry.attributes['value'], ref) # does nodeName even affect anything? entry.nodeName = entry.tagName = entry.endTagName = 'a' entry.attributes = InsensitiveDict({'name': anchor})
url1 = 'https://www.baidu.com' headers = { b'User-Agent': [ b'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36' ], b'Upgrade-Insecure-Requests': [b'1'], b'Accept': [ b'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' ], b'Accept-Encoding': [b'gzip, deflate, br'], b'Accept-Language': [b'zh-CN,zh;q=0.9'], b'Host': [b'www.baidu.com'], } headers1 = InsensitiveDict() headers1[ b'User-Agent'] = b'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36' headers1[b'Upgrade-Insecure-Requests'] = b'1' headers1[ b'Accept'] = b'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' headers1[b'Accept-Encoding'] = b'gzip, deflate, br' headers1[b'Accept-Language'] = b'zh-CN,zh;q=0.9' headers1[b'Host'] = b'www.baidu.com' headers2 = Headers(rawHeaders=headers) resposne1 = requests.get(url=url1) print(resposne1.text)
def __send_http_response(self, inh, request, response_msg): """ When we've know what message to send back to the HTTP client which has connected to the HTTP server, we issue this message back. @param inh: an inhabinant who will receive the response @type inh: AbstractInhabitant @param request: web server request. @type request: Request @param response_msg: response message to send. @type response_msg: AbstractMessage """ if not self.connection_alive: logger.debug( 'Connection died inside %r while trying ' 'to reply with %r', self, response_msg) # That's very very bad! We should put the message back to queue. self.tr_manager.post_message(response_msg) else: logger.debug('Sending response %r to %r', response_msg, request.requestHeaders) response_msg.in_transfer = True # Add the default server-specific headers, # then the message-specific headers. _headers = InsensitiveDict( dict(request.responseHeaders.getAllRawHeaders())) _pragma = ','.join( ifilter(None, [response_msg.get_pragma(), self.extra_pragma()])) HTTPTransport.update_headers(_headers, {'Pragma': [_pragma]}) HTTPTransport.update_headers(_headers, response_msg.get_headers()) for k, v_list in _headers.iteritems(): request.responseHeaders.setRawHeaders(k, v_list) logger.debug('New headers for %r are: %r', response_msg, request.responseHeaders) @contract_epydoc def produce_body_in_main(body, msg): """ @type body: col.Iterable @type msg: AbstractMessage """ assert in_main_thread() try: producer = LazyPullProducer(request, body) producer.completed.addBoth(self._ready_to_finish.callback) except Exception: logger.exception('Error during writing back the request:') # We haven't succeeded to send the message to the client. # Put it back to the queue. callInThread(self.tr_manager.post_message, msg) assert not in_main_thread() # get_body() may take long _body = response_msg.get_body() logger.debug('Writing body: %r', response_msg) callFromThread(produce_body_in_main, _body, response_msg)
def _render(self, request, avatar): """Node-specific request processing. @param request: Web server request. @type request: Request @type avatar: Host @returns: nothing (not a C{Deferred} object!). But, due to C{inlineCallbacks}, yields the intermediate C{Deferred}s. @todo: Proper error handling. @todo: Properly handle the situation when the passthrough is restricted: generate the reply which contains the Error 404. @todo: This should be streamlined using super() as soon as C{twisted.web.resource.Resource} becomes a new-style class """ assert not in_main_thread() # Kind-of-calling the parent, but the parent in this case should # return a Deferred as well,... let's not bother with it. # DeferredProcessingResource._render(self, request, avatar) logger.debug('') logger.debug('') logger.debug('Avatar %r;' '\tincoming headers %r;' '\tfrom %r', avatar, request.requestHeaders, request.client) # We setup the default headers for response even before # we start analyzing the incoming message, # so that if analyzing fails for some reason, we have the headers # and can create a more-or-less proper error reply. _requestHeaders = InsensitiveDict( dict(request.requestHeaders.getAllRawHeaders())) _responseHeaders = InsensitiveDict() self.initialize_headers(_responseHeaders) message = self.create_message_from_headers(headers=_requestHeaders, auth_peer=avatar) message.body_receiver = RequestBodyReceiver(message=message, request=request) # Make sure the body is received, before going further dummy_received_data = yield message.body_receiver.on_finish assert not in_main_thread() if message.direct: # If this is the message directed to ourselves, # process it and send the response logger.debug('') logger.debug('> Der Mitteilung Nahert: %r von %s', message, message.src) # Process the incoming message try: self.tr_manager.handle_incoming_message(message) except TransactionProcessingException as e: # The incoming transaction cannot be processed. # TODO: handle error properly, send a error reply maybe logger.error('Error (%r) during incoming msg processing: %s', e, e) logger.debug('Will send response to %r from %r %s', message, message.src, hash(message.src)) for k, v_list in _responseHeaders.iteritems(): request.responseHeaders.setRawHeaders(k, v_list) logger.debug('Done with response headers') elif self._accept_passthrough_message(message): # Non-direct message, and passthrough allowed: # wait for the body, then put in the queue. logger.debug('Passthrough message %r: body already available', message) self.tr_manager.post_message(message) else: # Passthrough is restricted for this message! raise NotImplementedError(u'Failed message {!r}:\n' u'from: {!r}\n' u' to: {!r}'.format( message, message.src, message.dst)) # How to check if we still need a message? still_need_checker = lambda: self.connection_alive # No matter if we are processing the message or passing it through, # we must send a response somehow. # If this is the message to us, we'd prefer a response to reply it; # but if this is a passthrough message, it will go to a different peer. prefer_msg_uuid = message.uuid if message.direct else None try: # Let's (asynchronously) wait for the response message here. resp_msg = self.__response_msg \ = yield self.tr_manager.wait_for_message_for_peer( inh=message.src, prefer_msg_uuid=prefer_msg_uuid, still_wait_checker=still_need_checker) except internet_error.ConnectionClosed as e: logger.warning('Stopped to wait for a message due to %r', e) self._handle_connection_close(Failure(e)) except Exception as e: logger.exception('Stopped to wait for a message due to %r', e) self._handle_connection_close(Failure(e)) else: # Since now and till the end of transfer we store the message # in self.__response_msg. assert not in_main_thread() if resp_msg is None: # The connection died already, while waiting for the message. assert not self.connection_alive else: # Got the message, send it back assert isinstance(resp_msg, AbstractMessage), repr(resp_msg) self.__send_http_response(message.src, request, resp_msg)
def _create_message_from_headers(self, headers, auth_peer, me, known_inhs, must_know_peer=True, peer_proto=None): """ Given the headers of the message, the "myself" reference to the receiving inhabitant, as well as the container of other known inhabitants, create a message. @param headers: The headers of the message. @type headers: InsensitiveDict @param auth_peer: What inhabitant authenticated as the originator of the message? @type auth_peer: NoneType, AbstractInhabitant @param me: The inhabitant object of the system processing this message. @type me: AbstractInhabitant @param known_inhs: The dictionary-like mapping the UUIDs to other known inhabitants of the system. @param must_know_peer: Whether we must know the peer beforehand to process the message. At the moment, it is used on the host, when it receives the messages from unknown other hosts. In the future, we ALWAYS must know the peer. @type must_know_peer: bool @todo: As soon as the Host->Host connections are preceded with the connection from the Node to the target Host (which notifies the target Host about the expected connection), C{must_know_peer} argument should be removed (and the code kept in the C{must_know_peer = True} position). @param peer_proto: The function which creates a peer object prototype depending upon the available information, if the peer is unknown. Default value is a subclass of C{AbstractInhabitant} constructor. Must accept all the usual C{AbstractInhabitant} constructor arguments. @todo: Remove peer_proto as long as must_know_peer is removed. @returns: The new message object. @rtype: AbstractMessage @raises MessageProcessingException: if the message could not be processed properly. """ if peer_proto is None: peer_proto = AbstractInhabitant # Validate HTTP request pragma_data_list = headers.get('Pragma', None) if pragma_data_list is None or not pragma_data_list: raise MessageProcessingException('No message type header') assert isinstance(pragma_data_list, list) if len(pragma_data_list) != 1: raise MessageProcessingException( 'Message type header is misconstructed') pragma_data = HTTPTransport.parse_pragma_header(pragma_data_list[0]) # Validate again if 'msgtype' not in pragma_data: raise MessageProcessingException('No message type') if 'msgid' not in pragma_data: raise MessageProcessingException('No message ID') if 'from' not in pragma_data: raise MessageProcessingException('No source peer UUID') if 'to' not in pragma_data: raise MessageProcessingException('No destination peer UUID') if 'status_code' not in pragma_data: logger.warning('No status code!') # TODO: after some time (when all clients have been migrated), # change it to "raise MessageProcessingException()" below. # raise MessageProcessingException('No status code') # Validate build version if 'build' in pragma_data: _build_str = pragma_data['build'] try: self.validate_build(_build_str) except MessageProcessingException: raise except Exception: logger.exception('Received the unparseable build version %r:', _build_str) raise MessageProcessingException( 'Incorrect build version: {}'.format(_build_str)) # Validate status code try: status_code_txt = pragma_data.get('status_code', 0) int(status_code_txt) except: logger.error('Received the unparseable status_code %r:', status_code_txt) # TODO: after some time (when all clients have been migrated), # change it to "raise MessageProcessingException() msg_type = pragma_data['msgtype'] msg_uuid = uuid.UUID(pragma_data['msgid']) _from_uuid = uuid.UUID(pragma_data['from']) to_uuid = uuid.UUID(pragma_data['to']) status_code = int(pragma_data.get('status_code', 0)) # Pre-validate origination peer if _from_uuid == NULL_UUID and auth_peer is None: raise MessageProcessingException( 'Missing source UUID: {}'.format(_from_uuid)) from_uuid = _from_uuid if _from_uuid != NULL_UUID \ else auth_peer.uuid # This still can be NULL_UUID at this point. if from_uuid == NULL_UUID: logger.debug('Totally no idea what host does %r have', auth_peer) # Validate termination UUID if me.uuid in (to_uuid, NULL_UUID): direct = True dst_inh = me elif to_uuid in known_inhs: direct = False dst_inh = known_inhs[to_uuid] logger.debug('Passthrough connection from %r to %r', from_uuid, to_uuid) else: raise MessageProcessingException( 'Unknown destination UUID: {}'.format(to_uuid)) # Now create the AbstractMessage object according # to the received information. # Some more validation of the origination peer if auth_peer is not None: logger.debug('Assuming sender/auth_peer is %r', auth_peer) src_inh = auth_peer elif from_uuid not in known_inhs and must_know_peer: raise MessageProcessingException( 'Unknown source UUID: {}'.format(from_uuid)) else: logger.debug('Assuming sender/uuid is %r', from_uuid) # Try to get the peer somehow src_inh = known_inhs.get(from_uuid, None) if src_inh is None: src_inh = peer_proto(uuid=from_uuid) logger.debug('Finally, sender is %r', src_inh) assert self.tr_manager is not None message = self.tr_manager.create_message(name=msg_type, src=src_inh, dst=dst_inh, uuid=msg_uuid, direct=direct, status_code=status_code) message.init_from_headers(InsensitiveDict(headers)) return message
def get_channel(self, channel): return self.channels.setdefault(channel, InsensitiveDict())