def _GetResponse( self, method_string, path_and_query, request_headers, body, attempt_number = 1 ): try: self._connection.request( method_string, path_and_query, headers = request_headers, body = body ) return self._connection.getresponse() except ( httplib.CannotSendRequest, httplib.BadStatusLine ): # for some reason, we can't send a request on the current connection, so let's make a new one and try again! time.sleep( 1 ) if attempt_number <= 3: self._RefreshConnection() return self._GetResponse( method_string, path_and_query, request_headers, body, attempt_number = attempt_number + 1 ) else: raise except socket.error as e: if e.errno == errno.WSAEACCES: text = 'The hydrus client did not have permission to make a connection to ' + HydrusData.ToUnicode( self._host ) if self._port is not None: text += ' on port ' + HydrusData.ToUnicode( self._port ) text += '. This is usually due to a firewall stopping it.' raise HydrusExceptions.FirewallException( text ) elif e.errno == errno.WSAECONNRESET: time.sleep( 5 ) if attempt_number <= 3: self._RefreshConnection() return self._GetResponse( method_string, path_and_query, request_headers, body, attempt_number = attempt_number + 1 ) else: text = 'The hydrus client\'s connection to ' + HydrusData.ToUnicode( self._host ) + ' kept on being reset by the remote host, so the attempt was abandoned.' raise HydrusExceptions.NetworkException( text ) else: raise
def _RefreshConnection( self ): if self._scheme == 'http': self._connection = httplib.HTTPConnection( self._host, self._port, timeout = self._timeout ) elif self._scheme == 'https': self._connection = httplib.HTTPSConnection( self._host, self._port, timeout = self._timeout ) try: self._connection.connect() except Exception as e: text = 'Could not connect to ' + HydrusData.ToUnicode( self._host ) + ':' text += os.linesep * 2 text += HydrusData.ToUnicode( e ) raise HydrusExceptions.NetworkException( text )
def _RefreshConnection(self): if self._scheme == 'http': self._connection = httplib.HTTPConnection(self._host, self._port, timeout=self._timeout) elif self._scheme == 'https': new_options = HydrusGlobals.client_controller.GetNewOptions() if self._hydrus_network or not new_options.GetBoolean( 'verify_regular_https'): # this negotiates decent encryption but won't check hostname or the certificate context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.options |= ssl.OP_NO_SSLv2 context.options |= ssl.OP_NO_SSLv3 self._connection = httplib.HTTPSConnection( self._host, self._port, timeout=self._timeout, context=context) else: self._connection = httplib.HTTPSConnection( self._host, self._port, timeout=self._timeout) try: self._connection.connect() except Exception as e: text = 'Could not connect to ' + HydrusData.ToUnicode( self._host) + ':' text += os.linesep * 2 text += HydrusData.ToUnicode(e) raise HydrusExceptions.NetworkException(text)
def _ReadResponse( self, response, report_hooks, temp_path = None ): try: if response.status == 200 and temp_path is not None: size_of_response = self._WriteResponseToPath( response, temp_path, report_hooks ) parsed_response = 'response written to temporary file' else: ( parsed_response, size_of_response ) = self._ParseResponse( response, report_hooks ) except socket.timeout as e: raise HydrusExceptions.NetworkException( 'Connection timed out during response read.' ) return ( parsed_response, size_of_response )
def Start(self): try: request_completed = False while not request_completed: try: response = self._SendRequestAndGetResponse() with self._lock: if self._body is not None: self._ReportDataUsed(len(self._body)) if 'Content-Type' in response.headers: self._content_type = response.headers['Content-Type'] if response.ok: with self._lock: self._status_text = u'downloading\u2026' if self._temp_path is None: self._ReadResponse(response, self._stream_io, 104857600) else: with open(self._temp_path, 'wb') as f: self._ReadResponse(response, f) with self._lock: self._status_text = 'done!' else: with self._lock: self._status_text = str( response.status_code) + ' - ' + str( response.reason) self._ReadResponse(response, self._stream_io, 104857600) with self._lock: self._stream_io.seek(0) data = self._stream_io.read() (e, error_text ) = ConvertStatusCodeAndDataIntoExceptionInfo( response.status_code, data, self.IS_HYDRUS_SERVICE) self._SetError(e, error_text) request_completed = True except HydrusExceptions.ShouldReattemptNetworkException as e: self._current_connection_attempt_number += 1 if not self._CanReattemptRequest(): raise HydrusExceptions.NetworkException( 'Ran out of reattempts on this error: ' + HydrusData.ToUnicode(e)) with self._lock: self._status_text = HydrusData.ToUnicode( e) + '--retrying' time.sleep(3) except requests.exceptions.ChunkedEncodingError: self._current_connection_attempt_number += 1 if not self._CanReattemptRequest(): raise HydrusExceptions.ConnectionException( 'Unable to complete request--it broke mid-way!') with self._lock: self._status_text = u'connection broke mid-request--retrying' time.sleep(3) except (requests.exceptions.ConnectionError, requests.exceptions.ConnectTimeout): self._current_connection_attempt_number += 1 if not self._CanReattemptConnection(): raise HydrusExceptions.ConnectionException( 'Could not connect!') with self._lock: self._status_text = u'connection failed--retrying' time.sleep(3) except requests.exceptions.ReadTimeout: self._current_connection_attempt_number += 1 if not self._CanReattemptRequest(): raise HydrusExceptions.ConnectionException( 'Connection successful, but reading response timed out!' ) with self._lock: self._status_text = u'read timed out--retrying' time.sleep(3) except Exception as e: with self._lock: self._status_text = 'unexpected error!' trace = traceback.format_exc() HydrusData.Print(trace) self._SetError(e, trace) finally: with self._lock: self._SetDone()
def _ReadResponse(self, response, stream_dest, max_allowed=None): with self._lock: if self._content_type is not None and self._content_type in HC.mime_enum_lookup: mime = HC.mime_enum_lookup[self._content_type] else: mime = None if 'content-length' in response.headers: self._num_bytes_to_read = int( response.headers['content-length']) if max_allowed is not None and self._num_bytes_to_read > max_allowed: raise HydrusExceptions.NetworkException( 'The url was apparently ' + HydrusData.ConvertIntToBytes(self._num_bytes_to_read) + ' but the max network size for this type of job is ' + HydrusData.ConvertIntToBytes(max_allowed) + '!') if self._file_import_options is not None: certain = True self._file_import_options.CheckNetworkDownload( mime, self._num_bytes_to_read, certain) else: self._num_bytes_to_read = None for chunk in response.iter_content(chunk_size=65536): if self._IsCancelled(): return stream_dest.write(chunk) chunk_length = len(chunk) with self._lock: self._num_bytes_read += chunk_length if max_allowed is not None and self._num_bytes_read > max_allowed: raise HydrusExceptions.NetworkException( 'The url exceeded the max network size for this type of job, which is ' + HydrusData.ConvertIntToBytes(max_allowed) + '!') if self._file_import_options is not None: certain = False self._file_import_options.CheckNetworkDownload( mime, self._num_bytes_to_read, certain) self._ReportDataUsed(chunk_length) self._WaitOnOngoingBandwidth() if HG.view_shutdown: raise HydrusExceptions.ShutdownException() if self._num_bytes_to_read is not None and self._num_bytes_read < self._num_bytes_to_read * 0.8: raise HydrusExceptions.ShouldReattemptNetworkException( 'Was expecting ' + HydrusData.ConvertIntToBytes(self._num_bytes_to_read) + ' but only got ' + HydrusData.ConvertIntToBytes(self._num_bytes_read) + '.')
def _GetInitialResponse(self, method, path_and_query, request_headers, body, attempt_number=1): if method == HC.GET: method_string = 'GET' elif method == HC.POST: method_string = 'POST' try: self._connection.request(method_string, path_and_query, headers=request_headers, body=body) return (self._connection.getresponse(), attempt_number) except (httplib.CannotSendRequest, httplib.BadStatusLine): # for some reason, we can't send a request on the current connection, so let's make a new one and try again! time.sleep(1) if attempt_number <= 3: self._RefreshConnection() return self._GetInitialResponse(method, path_and_query, request_headers, body, attempt_number=attempt_number + 1) else: raise except socket.error as e: if HC.PLATFORM_WINDOWS: access_errors = [errno.EACCES, errno.WSAEACCES] connection_reset_errors = [ errno.ECONNRESET, errno.WSAECONNRESET ] else: access_errors = [errno.EACCES] connection_reset_errors = [errno.ECONNRESET] if e.errno in access_errors: text = 'The hydrus client did not have permission to make a connection to ' + HydrusData.ToUnicode( self._host) if self._port is not None: text += ' on port ' + HydrusData.ToUnicode(self._port) text += '. This is usually due to a firewall stopping it.' raise HydrusExceptions.FirewallException(text) elif e.errno in connection_reset_errors: time.sleep(5) if attempt_number <= 3: self._RefreshConnection() return self._GetInitialResponse( method, path_and_query, request_headers, body, attempt_number=attempt_number + 1) else: text = 'The hydrus client\'s connection to ' + HydrusData.ToUnicode( self._host ) + ' kept on being reset by the remote host, so the attempt was abandoned.' raise HydrusExceptions.NetworkException(text) else: raise except ssl.SSLEOFError: time.sleep(5) if attempt_number <= 3: self._RefreshConnection() return self._GetInitialResponse(method_string, path_and_query, request_headers, body, attempt_number=attempt_number + 1) else: text = 'The hydrus client\'s ssl connection to ' + HydrusData.ToUnicode( self._host ) + ' kept terminating abruptly, so the attempt was abandoned.' raise HydrusExceptions.NetworkException(text)