def read_response(self): if not self._reader: raise ConnectionError("Socket closed on remote end") # _next_response might be cached from a can_read() call if self._next_response is not False: response = self._next_response self._next_response = False return response response = self._reader.gets() while response is False: try: buffer = self._sock.recv(4096) except (socket.error, socket.timeout): e = sys.exc_info()[1] raise ConnectionError("Error while reading from socket: %s" % (e.args, )) if not buffer: raise ConnectionError("Socket closed on remote end") self._reader.feed(buffer) # proactively, but not conclusively, check if more data is in the # buffer. if the data received doesn't end with \r\n, there's more. if not buffer.endswith(SYM_CRLF): continue response = self._reader.gets() if isinstance(response, ResponseError): response = self.parse_error(response.args[0]) # hiredis only knows about ResponseErrors. # self.parse_error() might turn the exception into a ConnectionError # which needs raising. if isinstance(response, ConnectionError): raise response return response
def _get(self, relative_url, **kwargs): url = urljoin(self.base_url, relative_url) logger.debug('Hitting url: %s with params: %s', url, kwargs) try: response = self.session.get(url, params=kwargs) except requests.ConnectionError: raise ConnectionError( 'Unable to connect to %s - are you sure the subdomain is correct?' % self.base_url) try: response.raise_for_status() except requests.HTTPError as e: if response.status_code == 401: raise ConnectionError( 'Unable to authenticate - are you sure the API key is correct?' ) elif response.status_code == 404: raise ConnectionError( 'No such resource %s - are you sure the endpoints were configured correctly?' % relative_url) else: raise (e) try: response_json = response.json() except ValueError: logger.exception('Response was not valid JSON!') return None if 'deleted_data' in response_json.keys(): rows = response_json['rowcount'] data = response_json['data'] assert len(data) == rows deleted_rows = response_json['deleted_rowcount'] deleted_data = response_json['deleted_data'] assert len(deleted_data) == deleted_rows logger.debug( 'Returned %s rows of data and %s rows of deleted data', rows, deleted_rows) return data, deleted_data elif 'rowcount' in response_json and 'data' in response_json: rows = response_json['rowcount'] data = response_json['data'] assert len(data) == rows logger.debug('Returned %s rows of data', rows) else: data = response_json return data
def read_response(self): if not self._reader: raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR) # _next_response might be cached from a can_read() call if self._next_response is not False: response = self._next_response self._next_response = False return response response = self._reader.gets() socket_read_size = self.socket_read_size while response is False: try: if HIREDIS_USE_BYTE_BUFFER: bufflen = recv_into(self._sock, self._buffer) if bufflen == 0: raise socket.error(SERVER_CLOSED_CONNECTION_ERROR) else: buffer = recv(self._sock, socket_read_size) # an empty string indicates the server shutdown the socket if not isinstance(buffer, bytes) or len(buffer) == 0: raise socket.error(SERVER_CLOSED_CONNECTION_ERROR) except socket.timeout: raise TimeoutError("Timeout reading from socket") except socket.error: e = sys.exc_info()[1] raise ConnectionError("Error while reading from socket: %s" % (e.args, )) if HIREDIS_USE_BYTE_BUFFER: self._reader.feed(self._buffer, 0, bufflen) else: self._reader.feed(buffer) response = self._reader.gets() # if an older version of hiredis is installed, we need to attempt # to convert ResponseErrors to their appropriate types. if not HIREDIS_SUPPORTS_CALLABLE_ERRORS: if isinstance(response, ResponseError): response = self.parse_error(response.args[0]) elif isinstance(response, list) and response and \ isinstance(response[0], ResponseError): response[0] = self.parse_error(response[0].args[0]) # if the response is a ConnectionError or the response is a list and # the first item is a ConnectionError, raise it as something bad # happened if isinstance(response, ConnectionError): raise response elif isinstance(response, list) and response and \ isinstance(response[0], ConnectionError): raise response[0] return response
def get_statement(self, refresh=False): if not refresh and self._statement: return self._statement if not self.dataverse: raise NoContainerError('This dataset has not been added to a Dataverse.') if not self.statement_uri: # Try to find statement uri without a request to the server link = get_element( self.get_entry(), tag='link', attribute='rel', attribute_value='http://purl.org/net/sword/terms/statement', ) if link is None: # Find link with request to server link = get_element( self.get_entry(refresh=True), tag='link', attribute='rel', attribute_value='http://purl.org/net/sword/terms/statement', ) self.statement_uri = link.get('href') resp = requests.get(self.statement_uri, auth=self.connection.auth) if resp.status_code != 200: raise ConnectionError('Statement could not be retrieved.') self._statement = resp.content return self._statement
def connect_to(self, address): self.host, self.port = address super(SentinelManagedConnection, self).connect() if self.connection_pool.check_connection: self.send_command('PING') if nativestr(self.read_response()) != 'PONG': raise ConnectionError('PING failed')
def send_packed_command(self, command): "Send an already packed command to the Redis server" if not self._sock: self.connect() try: if isinstance(command, str): command = [command] for item in command: self._sock.sendall(item) except socket.timeout: self.disconnect() raise TimeoutError("Timeout writing to socket") except socket.error: e = sys.exc_info()[1] self.disconnect() if len(e.args) == 1: errno, errmsg = 'UNKNOWN', e.args[0] else: errno = e.args[0] errmsg = e.args[1] raise ConnectionError("Error %s while writing to socket. %s." % (errno, errmsg)) except: self.disconnect() raise
def get_metadata(self, version='latest', refresh=False): if not refresh and self._metadata.get(version): return self._metadata[version] if not self.dataverse: raise NoContainerError('This dataset has not been added to a Dataverse.') url = 'https://{0}/api/datasets/{1}/versions/:{2}'.format( self.connection.host, self.id, version, ) resp = requests.get(url, params={'key': self.connection.token}) if resp.status_code == 404: raise VersionJsonNotFoundError('JSON metadata could not be found for this version.') elif resp.status_code != 200: raise ConnectionError('JSON metadata could not be retrieved.') metadata = resp.json()['data'] self._metadata[version] = metadata # Update corresponding version metadata if retrieving 'latest' if version == 'latest': latest_version = 'latest-published' if metadata['versionState'] == 'RELEASED' else 'draft' self._metadata[latest_version] = metadata return metadata
def can_read(self): if not self._reader: raise ConnectionError("Socket closed on remote end") if self._next_response is False: self._next_response = self._reader.gets() return self._next_response is not False
def write(self, data, try_left=None): if try_left is None: try_left = self.try_left if not self._stream: self.connect() if not self._stream: raise ConnectionError('Tried to write to non-existent connection') if try_left > 0: try: self._stream.write(data) except IOError: self.disconnect() self.write(data, try_left - 1) else: raise ConnectionError('Tried to write to non-existent connection')
def _read_from_socket(self, length=None): socket_read_size = self.socket_read_size buf = self._buffer buf.seek(self.bytes_written) marker = 0 try: while True: data = recv(self._sock, socket_read_size) # an empty string indicates the server shutdown the socket if isinstance(data, bytes) and len(data) == 0: raise socket.error(SERVER_CLOSED_CONNECTION_ERROR) buf.write(data) data_length = len(data) self.bytes_written += data_length marker += data_length if length is not None and length > marker: continue break except socket.timeout: raise TimeoutError("Timeout reading from socket") except socket.error: e = sys.exc_info()[1] raise ConnectionError("Error while reading from socket: %s" % (e.args, ))
def connect (self): ftp = self.ftp = ftplib.FTP() try: ftp.connect(self.host) except socket.error: raise ConnectionError("Connecting to FTP server failed") try: ftp.login(self.username, self.password) except ftplib.error_perm: raise ConnectionError("Authentication failed") if self.root: if not self.root.startswith("/"): self.root = "/" + self.root self.cd(self.root) self.cdRoot()
def can_read(self): if not self._reader: raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR) if self._next_response is False: self._next_response = self._reader.gets() return self._next_response is not False
def get_connection(self, command_name, *keys, **options): """ Get a connection, blocking for ``self.timeout`` until a connection is available from the pool. If the connection returned is ``None`` then creates a new connection. Because we use a last-in first-out queue, the existing connections (having been returned to the pool after the initial ``None`` values were added) will be returned before ``None`` values. This means we only create new connections when we need to, i.e.: the actual number of connections will only increase in response to demand. """ # Make sure we haven't changed process. self._checkpid() # Try and get a connection from the pool. If one isn't available within # self.timeout then raise a ``ConnectionError``. connection = None try: connection = self.pool.get(block=True, timeout=self.timeout) except Empty: # Note that this is not caught by the redis client and will be # raised unless handled by application code. If you want never to raise ConnectionError("No connection available.") # If the ``connection`` is actually ``None`` then that's a cue to make # a new connection to add to the pool. if connection is None: connection = self.make_connection() return connection
def read(self, length, callback): try: if not self._stream: self.disconnect() raise ConnectionError('Tried to read from non-existent connection') self._stream.read_bytes(length, callback) except IOError: self.fire_event('on_disconnect')
def _get(self, relative_url, *args, **kwargs): """ """ for arg in args: relative_url += '/{}'.format(arg) url = urljoin(self.base_url, relative_url) logger.debug('Hitting url: %s with params: %s', url, kwargs) try: response = self.session.get(url, params=kwargs) except requests.ConnectionError: raise ConnectionError( 'Unable to connect to %s - are you sure the subdomain is correct?' % self.base_url) try: response.raise_for_status() except requests.HTTPError as e: if response.status_code == 401: raise ConnectionError( 'Unable to authenticate - are you sure the API key is correct?' ) elif response.status_code == 404: raise ConnectionError( 'No such resource %s - are you sure the endpoints were configured correctly?' % relative_url) else: raise (e) try: response_json = response.json() except ValueError: logger.exception('Response was not valid JSON!') return None if args: data = response_json else: rows = response_json['rowcount'] data = response_json['data'] assert len(data) == rows return data
def readline(self, callback): try: if not self._stream: self.disconnect() raise ConnectionError('Tried to read from non-existent connection') saved_callback = stack_context.wrap(callback) self.read_callbacks.append(saved_callback) self._stream.read_until('\r\n', callback=partial(self.readline_callback, saved_callback)) except IOError: self.fire_event('on_disconnect')
def connect(self): """Connect to the server :raise ConnectionError: If socket cannot establish a connection """ try: logger.info(u'Connecting %s:%d' % (self.host, self.port)) self.sock.connect((self.host, self.port)) except socket.error: raise ConnectionError() self.state = CONNECTED
def read(self, length, callback=None): try: if not self._stream: self.disconnect() raise ConnectionError('Tried to read from ' 'non-existent connection') self.read_callbacks.append(callback) self._stream.read_bytes(length, callback=partial(self.read_callback, callback)) except IOError: self.fire_event('on_disconnect')
def get_entry(self, refresh=False): if not refresh and self._entry is not None: return etree.tostring(self._entry) resp = requests.get(self.edit_uri, auth=self.connection.auth) if resp.status_code != 200: raise ConnectionError('Atom entry could not be retrieved.') entry_string = resp.content self._entry = etree.XML(entry_string) return entry_string
def connect(self): if not self._stream: try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) sock.settimeout(self.timeout) sock.connect((self.host, self.port)) self._stream = IOStream(sock, io_loop=self._io_loop) self._stream.set_close_callback(self.on_stream_close) self.connected() except socket.error, e: raise ConnectionError(str(e)) self.fire_event('on_connect')
def on_connect(self): "Initialize the connection, authenticate and select a database" self._parser.on_connect(self) # if a password is specified, authenticate if self.password: self.send_command('AUTH', self.password) if nativestr(self.read_response()) != 'OK': raise AuthenticationError('Invalid Password') # if a database is specified, switch to it if self.db: self.send_command('SELECT', self.db) if nativestr(self.read_response()) != 'OK': raise ConnectionError('Invalid Database')
def get_contents(self, refresh=False): if not refresh and self._contents_json: return self._contents_json content_uri = 'https://{0}/api/dataverses/{1}/contents'.format( self.connection.host, self.alias ) resp = requests.get( content_uri, params={'key': self.connection.token} ) if resp.status_code != 200: raise ConnectionError('Atom entry could not be retrieved.') self._contents_json = resp.json()['data'] return self._contents_json
def read_response(self): response = self._buffer.readline() if not response: raise ConnectionError("Socket closed on remote end") byte, response = byte_to_chr(response[0]), response[1:] if byte not in ('-', '+', ':', '$', '*'): raise InvalidResponse("Protocol Error: %s, %s" % (str(byte), str(response))) # server returned an error if byte == '-': response = nativestr(response) error = self.parse_error(response) # if the error is a ConnectionError, raise immediately so the user # is notified if isinstance(error, ConnectionError): raise error # otherwise, we're dealing with a ResponseError that might belong # inside a pipeline response. the connection's read_response() # and/or the pipeline's execute() will raise this error if # necessary, so just return the exception instance here. return error # single value elif byte == '+': pass # int value elif byte == ':': response = long(response) # bulk response elif byte == '$': length = int(response) if length == -1: return None response = self._buffer.read(length) # multi-bulk response elif byte == '*': length = int(response) if length == -1: return None response = [self.read_response() for i in xrange(length)] if isinstance(response, bytes) and self.encoding: response = response.decode(self.encoding) return response
def send_packed_command(self, command): "Send an already packed command to the Redis server" if not self._sock: self.connect() try: self._sock.sendall(command) except socket.error: e = sys.exc_info()[1] self.disconnect() if len(e.args) == 1: _errno, errmsg = 'UNKNOWN', e.args[0] else: _errno, errmsg = e.args raise ConnectionError("Error %s while writing to socket. %s." % (_errno, errmsg)) except: self.disconnect() raise
def _read_from_socket(self, length=None): chunksize = 8192 if length is None: length = chunksize buf = self._buffer buf.seek(self.bytes_written) marker = 0 try: while length > marker: data = self._sock.recv(chunksize) buf.write(data) self.bytes_written += len(data) marker += chunksize except (socket.error, socket.timeout): e = sys.exc_info()[1] raise ConnectionError("Error while reading from socket: %s" % (e.args, ))
def execute(self, callback=None): command_stack = self.command_stack self.command_stack = [] if self.transactional: command_stack = [CmdLine('MULTI')] \ + command_stack \ + [CmdLine('EXEC')] request = format_pipeline_request(command_stack) try: self.connection.write(request) except IOError: self.command_stack = [] self.connection.disconnect() raise ConnectionError("Socket closed on remote end") except Exception, e: self.command_stack = [] self.connection.disconnect() raise e
def connect(self): "Connects to the Redis server if not already connected" if self._sock: return try: sock = self._connect() except socket.error: e = sys.exc_info()[1] raise ConnectionError(self._error_message(e)) self._sock = sock try: self.on_connect() except RedisError: # clean up after any error in on_connect self.disconnect() raise # run any user callbacks. right now the only internal callback # is for pubsub channel/pattern resubscription for callback in self._connect_callbacks: callback(self)
def error_wrapper(e): if isinstance(e, GeneratorExit): return ConnectionError('Connection lost') else: return e
def on_disconnect(self): if self.subscribed: self.subscribed = False raise ConnectionError("Socket closed on remote end")
args.reverse() args.extend(bits) args.reverse() self.connection.write(self.format_command(*args, **kwargs)) except Exception, e: self.connection.disconnect() raise e if (cmd in PUB_SUB_COMMANDS) or (self.subscribed and cmd == 'PUBLISH'): result = True else: with self.connection as c: result = None data = yield gen.Task(c.readline) if not data: raise ConnectionError('no data received') else: resp = yield gen.Task(self.process_data, data, cmd_line) result = self.format_reply(cmd_line, resp) if callback: callback(result) @gen.engine def process_data(self, data, cmd_line, callback=None): data = data[:-2] # strip \r\n if data == '$-1': response = None elif data == '*0' or data == '*-1': response = [] else: