def test_add(self):
h = Headers({'header1': 'value1'})
h.add('header1', 'value3')
self.assertEqual(h.getlist('header1'), ['value1', 'value3'])
h = Headers()
h.add('header1', 'value1')
h.add('header1', 'value3')
self.assertEqual(h.getlist('header1'), ['value1', 'value3'])
def test_add(self):
h1 = Headers({'header1': 'value1'})
h1.add('header1', 'value3')
self.assertEqual(h1.getlist('header1'), ['value1', 'value3'])
h1 = Headers()
h1.add('header1', 'value1')
h1.add('header1', 'value3')
self.assertEqual(h1.getlist('header1'), ['value1', 'value3'])
class CrawlmiHTTPClient(HTTPClient):
delimiter = '\n'
def __init__(self):
self.body_size = 0
def connectionMade(self):
self.headers = Headers()
# method command
self.sendCommand(self.factory.method, self.factory.path)
# headers
for key, values in self.factory.headers.iteritems():
for value in values:
self.sendHeader(key, value)
self.endHeaders()
# body
if self.factory.body is not None:
self.transport.write(self.factory.body)
def extractHeader(self, header):
key, val = header.split(':', 1)
val = val.lstrip()
self.handleHeader(key, val)
if key.lower() == 'content-length':
self.length = int(val)
def lineReceived(self, line):
try:
HTTPClient.lineReceived(self, line.rstrip())
except:
self.factory.invalid_headers.append(line)
if hasattr(self.transport, 'abortConnection'):
self.transport.abortConnection()
else:
self.transport.loseConnection()
def handleHeader(self, key, value):
self.headers.add(key, value)
def handleStatus(self, version, status, message):
self.factory.gotStatus(version, status, message)
def handleEndHeaders(self):
self.factory.gotHeaders(self.headers)
def connectionLost(self, reason):
self._connection_lost_reason = reason
HTTPClient.connectionLost(self, reason)
self.factory.noPage(reason)
def handleResponse(self, response):
if self.factory.method.upper() == 'HEAD':
self.factory.page('')
elif self.length is not None and self.length > 0:
self.factory.noPage(self._connection_lost_reason)
else:
self.factory.page(response)
self.transport.loseConnection()
def timeout(self):
self.transport.loseConnection()
self.factory.noPage(
defer.TimeoutError('Getting %s took longer than %s seconds.' %
(self.factory.url, self.factory.timeout)))
def handleResponsePart(self, data):
HTTPClient.handleResponsePart(self, data)
self.body_size += len(data)
if (self.factory.download_size and
self.body_size > self.factory.download_size):
self.transport.loseConnection()
self.factory.noPage(
DownloadSizeError('Response exceeded %s bytes.' %
self.factory.download_size))
class CrawlmiHTTPClient(HTTPClient):
delimiter = '\n'
def __init__(self):
self.body_size = 0
def connectionMade(self):
self.headers = Headers()
# method command
self.sendCommand(self.factory.method, self.factory.path)
# headers
for key, values in self.factory.headers.iteritems():
for value in values:
self.sendHeader(key, value)
self.endHeaders()
# body
if self.factory.body is not None:
self.transport.write(self.factory.body)
def extractHeader(self, header):
key, val = header.split(':', 1)
val = val.lstrip()
self.handleHeader(key, val)
if key.lower() == 'content-length':
self.length = int(val)
def lineReceived(self, line):
try:
HTTPClient.lineReceived(self, line.rstrip())
except:
self.factory.invalid_headers.append(line)
if hasattr(self.transport, 'abortConnection'):
self.transport.abortConnection()
else:
self.transport.loseConnection()
def handleHeader(self, key, value):
self.headers.add(key, value)
def handleStatus(self, version, status, message):
self.factory.gotStatus(version, status, message)
def handleEndHeaders(self):
self.factory.gotHeaders(self.headers)
def connectionLost(self, reason):
self._connection_lost_reason = reason
HTTPClient.connectionLost(self, reason)
self.factory.noPage(reason)
def handleResponse(self, response):
if self.factory.method.upper() == 'HEAD':
self.factory.page('')
elif self.length is not None and self.length > 0:
self.factory.noPage(self._connection_lost_reason)
else:
self.factory.page(response)
self.transport.loseConnection()
def timeout(self):
self.transport.loseConnection()
self.factory.noPage(
defer.TimeoutError('Getting %s took longer than %s seconds.' %
(self.factory.url, self.factory.timeout)))
def handleResponsePart(self, data):
HTTPClient.handleResponsePart(self, data)
self.body_size += len(data)
if (self.factory.download_size
and self.body_size > self.factory.download_size):
self.transport.loseConnection()
self.factory.noPage(
DownloadSizeError('Response exceeded %s bytes.' %
self.factory.download_size))
