def test_add(self): h = Headers({'header1': 'value1'}) h.add('header1', 'value3') self.assertEqual(h.getlist('header1'), ['value1', 'value3']) h = Headers() h.add('header1', 'value1') h.add('header1', 'value3') self.assertEqual(h.getlist('header1'), ['value1', 'value3'])
def test_add(self): h1 = Headers({'header1': 'value1'}) h1.add('header1', 'value3') self.assertEqual(h1.getlist('header1'), ['value1', 'value3']) h1 = Headers() h1.add('header1', 'value1') h1.add('header1', 'value3') self.assertEqual(h1.getlist('header1'), ['value1', 'value3'])
class CrawlmiHTTPClient(HTTPClient): delimiter = '\n' def __init__(self): self.body_size = 0 def connectionMade(self): self.headers = Headers() # method command self.sendCommand(self.factory.method, self.factory.path) # headers for key, values in self.factory.headers.iteritems(): for value in values: self.sendHeader(key, value) self.endHeaders() # body if self.factory.body is not None: self.transport.write(self.factory.body) def extractHeader(self, header): key, val = header.split(':', 1) val = val.lstrip() self.handleHeader(key, val) if key.lower() == 'content-length': self.length = int(val) def lineReceived(self, line): try: HTTPClient.lineReceived(self, line.rstrip()) except: self.factory.invalid_headers.append(line) if hasattr(self.transport, 'abortConnection'): self.transport.abortConnection() else: self.transport.loseConnection() def handleHeader(self, key, value): self.headers.add(key, value) def handleStatus(self, version, status, message): self.factory.gotStatus(version, status, message) def handleEndHeaders(self): self.factory.gotHeaders(self.headers) def connectionLost(self, reason): self._connection_lost_reason = reason HTTPClient.connectionLost(self, reason) self.factory.noPage(reason) def handleResponse(self, response): if self.factory.method.upper() == 'HEAD': self.factory.page('') elif self.length is not None and self.length > 0: self.factory.noPage(self._connection_lost_reason) else: self.factory.page(response) self.transport.loseConnection() def timeout(self): self.transport.loseConnection() self.factory.noPage( defer.TimeoutError('Getting %s took longer than %s seconds.' % (self.factory.url, self.factory.timeout))) def handleResponsePart(self, data): HTTPClient.handleResponsePart(self, data) self.body_size += len(data) if (self.factory.download_size and self.body_size > self.factory.download_size): self.transport.loseConnection() self.factory.noPage( DownloadSizeError('Response exceeded %s bytes.' % self.factory.download_size))