def _GetResponse( self, method_string, path_and_query, request_headers, body, attempt_number = 1 ):
try:
self._connection.request( method_string, path_and_query, headers = request_headers, body = body )
return self._connection.getresponse()
except ( httplib.CannotSendRequest, httplib.BadStatusLine ):
# for some reason, we can't send a request on the current connection, so let's make a new one and try again!
time.sleep( 1 )
if attempt_number <= 3:
self._RefreshConnection()
return self._GetResponse( method_string, path_and_query, request_headers, body, attempt_number = attempt_number + 1 )
else:
raise
except socket.error as e:
if e.errno == errno.WSAEACCES:
text = 'The hydrus client did not have permission to make a connection to ' + HydrusData.ToUnicode( self._host )
if self._port is not None:
text += ' on port ' + HydrusData.ToUnicode( self._port )
text += '. This is usually due to a firewall stopping it.'
raise HydrusExceptions.FirewallException( text )
elif e.errno == errno.WSAECONNRESET:
time.sleep( 5 )
if attempt_number <= 3:
self._RefreshConnection()
return self._GetResponse( method_string, path_and_query, request_headers, body, attempt_number = attempt_number + 1 )
else:
text = 'The hydrus client\'s connection to ' + HydrusData.ToUnicode( self._host ) + ' kept on being reset by the remote host, so the attempt was abandoned.'
raise HydrusExceptions.NetworkException( text )
else:
raise
def _RefreshConnection( self ):
if self._scheme == 'http': self._connection = httplib.HTTPConnection( self._host, self._port, timeout = self._timeout )
elif self._scheme == 'https': self._connection = httplib.HTTPSConnection( self._host, self._port, timeout = self._timeout )
try:
self._connection.connect()
except Exception as e:
text = 'Could not connect to ' + HydrusData.ToUnicode( self._host ) + ':'
text += os.linesep * 2
text += HydrusData.ToUnicode( e )
raise HydrusExceptions.NetworkException( text )
def _RefreshConnection(self):
if self._scheme == 'http':
self._connection = httplib.HTTPConnection(self._host,
self._port,
timeout=self._timeout)
elif self._scheme == 'https':
new_options = HydrusGlobals.client_controller.GetNewOptions()
if self._hydrus_network or not new_options.GetBoolean(
'verify_regular_https'):
# this negotiates decent encryption but won't check hostname or the certificate
context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
context.options |= ssl.OP_NO_SSLv2
context.options |= ssl.OP_NO_SSLv3
self._connection = httplib.HTTPSConnection(
self._host,
self._port,
timeout=self._timeout,
context=context)
else:
self._connection = httplib.HTTPSConnection(
self._host, self._port, timeout=self._timeout)
try:
self._connection.connect()
except Exception as e:
text = 'Could not connect to ' + HydrusData.ToUnicode(
self._host) + ':'
text += os.linesep * 2
text += HydrusData.ToUnicode(e)
raise HydrusExceptions.NetworkException(text)
def _ReadResponse( self, response, report_hooks, temp_path = None ):
try:
if response.status == 200 and temp_path is not None:
size_of_response = self._WriteResponseToPath( response, temp_path, report_hooks )
parsed_response = 'response written to temporary file'
else:
( parsed_response, size_of_response ) = self._ParseResponse( response, report_hooks )
except socket.timeout as e:
raise HydrusExceptions.NetworkException( 'Connection timed out during response read.' )
return ( parsed_response, size_of_response )
def Start(self):
try:
request_completed = False
while not request_completed:
try:
response = self._SendRequestAndGetResponse()
with self._lock:
if self._body is not None:
self._ReportDataUsed(len(self._body))
if 'Content-Type' in response.headers:
self._content_type = response.headers['Content-Type']
if response.ok:
with self._lock:
self._status_text = u'downloading\u2026'
if self._temp_path is None:
self._ReadResponse(response, self._stream_io,
104857600)
else:
with open(self._temp_path, 'wb') as f:
self._ReadResponse(response, f)
with self._lock:
self._status_text = 'done!'
else:
with self._lock:
self._status_text = str(
response.status_code) + ' - ' + str(
response.reason)
self._ReadResponse(response, self._stream_io,
104857600)
with self._lock:
self._stream_io.seek(0)
data = self._stream_io.read()
(e, error_text
) = ConvertStatusCodeAndDataIntoExceptionInfo(
response.status_code, data,
self.IS_HYDRUS_SERVICE)
self._SetError(e, error_text)
request_completed = True
except HydrusExceptions.ShouldReattemptNetworkException as e:
self._current_connection_attempt_number += 1
if not self._CanReattemptRequest():
raise HydrusExceptions.NetworkException(
'Ran out of reattempts on this error: ' +
HydrusData.ToUnicode(e))
with self._lock:
self._status_text = HydrusData.ToUnicode(
e) + '--retrying'
time.sleep(3)
except requests.exceptions.ChunkedEncodingError:
self._current_connection_attempt_number += 1
if not self._CanReattemptRequest():
raise HydrusExceptions.ConnectionException(
'Unable to complete request--it broke mid-way!')
with self._lock:
self._status_text = u'connection broke mid-request--retrying'
time.sleep(3)
except (requests.exceptions.ConnectionError,
requests.exceptions.ConnectTimeout):
self._current_connection_attempt_number += 1
if not self._CanReattemptConnection():
raise HydrusExceptions.ConnectionException(
'Could not connect!')
with self._lock:
self._status_text = u'connection failed--retrying'
time.sleep(3)
except requests.exceptions.ReadTimeout:
self._current_connection_attempt_number += 1
if not self._CanReattemptRequest():
raise HydrusExceptions.ConnectionException(
'Connection successful, but reading response timed out!'
)
with self._lock:
self._status_text = u'read timed out--retrying'
time.sleep(3)
except Exception as e:
with self._lock:
self._status_text = 'unexpected error!'
trace = traceback.format_exc()
HydrusData.Print(trace)
self._SetError(e, trace)
finally:
with self._lock:
self._SetDone()
def _ReadResponse(self, response, stream_dest, max_allowed=None):
with self._lock:
if self._content_type is not None and self._content_type in HC.mime_enum_lookup:
mime = HC.mime_enum_lookup[self._content_type]
else:
mime = None
if 'content-length' in response.headers:
self._num_bytes_to_read = int(
response.headers['content-length'])
if max_allowed is not None and self._num_bytes_to_read > max_allowed:
raise HydrusExceptions.NetworkException(
'The url was apparently ' +
HydrusData.ConvertIntToBytes(self._num_bytes_to_read) +
' but the max network size for this type of job is ' +
HydrusData.ConvertIntToBytes(max_allowed) + '!')
if self._file_import_options is not None:
certain = True
self._file_import_options.CheckNetworkDownload(
mime, self._num_bytes_to_read, certain)
else:
self._num_bytes_to_read = None
for chunk in response.iter_content(chunk_size=65536):
if self._IsCancelled():
return
stream_dest.write(chunk)
chunk_length = len(chunk)
with self._lock:
self._num_bytes_read += chunk_length
if max_allowed is not None and self._num_bytes_read > max_allowed:
raise HydrusExceptions.NetworkException(
'The url exceeded the max network size for this type of job, which is '
+ HydrusData.ConvertIntToBytes(max_allowed) + '!')
if self._file_import_options is not None:
certain = False
self._file_import_options.CheckNetworkDownload(
mime, self._num_bytes_to_read, certain)
self._ReportDataUsed(chunk_length)
self._WaitOnOngoingBandwidth()
if HG.view_shutdown:
raise HydrusExceptions.ShutdownException()
if self._num_bytes_to_read is not None and self._num_bytes_read < self._num_bytes_to_read * 0.8:
raise HydrusExceptions.ShouldReattemptNetworkException(
'Was expecting ' +
HydrusData.ConvertIntToBytes(self._num_bytes_to_read) +
' but only got ' +
HydrusData.ConvertIntToBytes(self._num_bytes_read) + '.')
def _GetInitialResponse(self,
method,
path_and_query,
request_headers,
body,
attempt_number=1):
if method == HC.GET: method_string = 'GET'
elif method == HC.POST: method_string = 'POST'
try:
self._connection.request(method_string,
path_and_query,
headers=request_headers,
body=body)
return (self._connection.getresponse(), attempt_number)
except (httplib.CannotSendRequest, httplib.BadStatusLine):
# for some reason, we can't send a request on the current connection, so let's make a new one and try again!
time.sleep(1)
if attempt_number <= 3:
self._RefreshConnection()
return self._GetInitialResponse(method,
path_and_query,
request_headers,
body,
attempt_number=attempt_number +
1)
else:
raise
except socket.error as e:
if HC.PLATFORM_WINDOWS:
access_errors = [errno.EACCES, errno.WSAEACCES]
connection_reset_errors = [
errno.ECONNRESET, errno.WSAECONNRESET
]
else:
access_errors = [errno.EACCES]
connection_reset_errors = [errno.ECONNRESET]
if e.errno in access_errors:
text = 'The hydrus client did not have permission to make a connection to ' + HydrusData.ToUnicode(
self._host)
if self._port is not None:
text += ' on port ' + HydrusData.ToUnicode(self._port)
text += '. This is usually due to a firewall stopping it.'
raise HydrusExceptions.FirewallException(text)
elif e.errno in connection_reset_errors:
time.sleep(5)
if attempt_number <= 3:
self._RefreshConnection()
return self._GetInitialResponse(
method,
path_and_query,
request_headers,
body,
attempt_number=attempt_number + 1)
else:
text = 'The hydrus client\'s connection to ' + HydrusData.ToUnicode(
self._host
) + ' kept on being reset by the remote host, so the attempt was abandoned.'
raise HydrusExceptions.NetworkException(text)
else:
raise
except ssl.SSLEOFError:
time.sleep(5)
if attempt_number <= 3:
self._RefreshConnection()
return self._GetInitialResponse(method_string,
path_and_query,
request_headers,
body,
attempt_number=attempt_number +
1)
else:
text = 'The hydrus client\'s ssl connection to ' + HydrusData.ToUnicode(
self._host
) + ' kept terminating abruptly, so the attempt was abandoned.'
raise HydrusExceptions.NetworkException(text)
