def read_response(self):
if not self._reader:
raise ConnectionError("Socket closed on remote end")
# _next_response might be cached from a can_read() call
if self._next_response is not False:
response = self._next_response
self._next_response = False
return response
response = self._reader.gets()
while response is False:
try:
buffer = self._sock.recv(4096)
except (socket.error, socket.timeout):
e = sys.exc_info()[1]
raise ConnectionError("Error while reading from socket: %s" %
(e.args, ))
if not buffer:
raise ConnectionError("Socket closed on remote end")
self._reader.feed(buffer)
# proactively, but not conclusively, check if more data is in the
# buffer. if the data received doesn't end with \r\n, there's more.
if not buffer.endswith(SYM_CRLF):
continue
response = self._reader.gets()
if isinstance(response, ResponseError):
response = self.parse_error(response.args[0])
# hiredis only knows about ResponseErrors.
# self.parse_error() might turn the exception into a ConnectionError
# which needs raising.
if isinstance(response, ConnectionError):
raise response
return response
def _get(self, relative_url, **kwargs):
url = urljoin(self.base_url, relative_url)
logger.debug('Hitting url: %s with params: %s', url, kwargs)
try:
response = self.session.get(url, params=kwargs)
except requests.ConnectionError:
raise ConnectionError(
'Unable to connect to %s - are you sure the subdomain is correct?'
% self.base_url)
try:
response.raise_for_status()
except requests.HTTPError as e:
if response.status_code == 401:
raise ConnectionError(
'Unable to authenticate - are you sure the API key is correct?'
)
elif response.status_code == 404:
raise ConnectionError(
'No such resource %s - are you sure the endpoints were configured correctly?'
% relative_url)
else:
raise (e)
try:
response_json = response.json()
except ValueError:
logger.exception('Response was not valid JSON!')
return None
if 'deleted_data' in response_json.keys():
rows = response_json['rowcount']
data = response_json['data']
assert len(data) == rows
deleted_rows = response_json['deleted_rowcount']
deleted_data = response_json['deleted_data']
assert len(deleted_data) == deleted_rows
logger.debug(
'Returned %s rows of data and %s rows of deleted data', rows,
deleted_rows)
return data, deleted_data
elif 'rowcount' in response_json and 'data' in response_json:
rows = response_json['rowcount']
data = response_json['data']
assert len(data) == rows
logger.debug('Returned %s rows of data', rows)
else:
data = response_json
return data
def read_response(self):
if not self._reader:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
# _next_response might be cached from a can_read() call
if self._next_response is not False:
response = self._next_response
self._next_response = False
return response
response = self._reader.gets()
socket_read_size = self.socket_read_size
while response is False:
try:
if HIREDIS_USE_BYTE_BUFFER:
bufflen = recv_into(self._sock, self._buffer)
if bufflen == 0:
raise socket.error(SERVER_CLOSED_CONNECTION_ERROR)
else:
buffer = recv(self._sock, socket_read_size)
# an empty string indicates the server shutdown the socket
if not isinstance(buffer, bytes) or len(buffer) == 0:
raise socket.error(SERVER_CLOSED_CONNECTION_ERROR)
except socket.timeout:
raise TimeoutError("Timeout reading from socket")
except socket.error:
e = sys.exc_info()[1]
raise ConnectionError("Error while reading from socket: %s" %
(e.args, ))
if HIREDIS_USE_BYTE_BUFFER:
self._reader.feed(self._buffer, 0, bufflen)
else:
self._reader.feed(buffer)
response = self._reader.gets()
# if an older version of hiredis is installed, we need to attempt
# to convert ResponseErrors to their appropriate types.
if not HIREDIS_SUPPORTS_CALLABLE_ERRORS:
if isinstance(response, ResponseError):
response = self.parse_error(response.args[0])
elif isinstance(response, list) and response and \
isinstance(response[0], ResponseError):
response[0] = self.parse_error(response[0].args[0])
# if the response is a ConnectionError or the response is a list and
# the first item is a ConnectionError, raise it as something bad
# happened
if isinstance(response, ConnectionError):
raise response
elif isinstance(response, list) and response and \
isinstance(response[0], ConnectionError):
raise response[0]
return response
def get_statement(self, refresh=False):
if not refresh and self._statement:
return self._statement
if not self.dataverse:
raise NoContainerError('This dataset has not been added to a Dataverse.')
if not self.statement_uri:
# Try to find statement uri without a request to the server
link = get_element(
self.get_entry(),
tag='link',
attribute='rel',
attribute_value='http://purl.org/net/sword/terms/statement',
)
if link is None:
# Find link with request to server
link = get_element(
self.get_entry(refresh=True),
tag='link',
attribute='rel',
attribute_value='http://purl.org/net/sword/terms/statement',
)
self.statement_uri = link.get('href')
resp = requests.get(self.statement_uri, auth=self.connection.auth)
if resp.status_code != 200:
raise ConnectionError('Statement could not be retrieved.')
self._statement = resp.content
return self._statement
def connect_to(self, address):
self.host, self.port = address
super(SentinelManagedConnection, self).connect()
if self.connection_pool.check_connection:
self.send_command('PING')
if nativestr(self.read_response()) != 'PONG':
raise ConnectionError('PING failed')
def send_packed_command(self, command):
"Send an already packed command to the Redis server"
if not self._sock:
self.connect()
try:
if isinstance(command, str):
command = [command]
for item in command:
self._sock.sendall(item)
except socket.timeout:
self.disconnect()
raise TimeoutError("Timeout writing to socket")
except socket.error:
e = sys.exc_info()[1]
self.disconnect()
if len(e.args) == 1:
errno, errmsg = 'UNKNOWN', e.args[0]
else:
errno = e.args[0]
errmsg = e.args[1]
raise ConnectionError("Error %s while writing to socket. %s." %
(errno, errmsg))
except:
self.disconnect()
raise
def get_metadata(self, version='latest', refresh=False):
if not refresh and self._metadata.get(version):
return self._metadata[version]
if not self.dataverse:
raise NoContainerError('This dataset has not been added to a Dataverse.')
url = 'https://{0}/api/datasets/{1}/versions/:{2}'.format(
self.connection.host,
self.id,
version,
)
resp = requests.get(url, params={'key': self.connection.token})
if resp.status_code == 404:
raise VersionJsonNotFoundError('JSON metadata could not be found for this version.')
elif resp.status_code != 200:
raise ConnectionError('JSON metadata could not be retrieved.')
metadata = resp.json()['data']
self._metadata[version] = metadata
# Update corresponding version metadata if retrieving 'latest'
if version == 'latest':
latest_version = 'latest-published' if metadata['versionState'] == 'RELEASED' else 'draft'
self._metadata[latest_version] = metadata
return metadata
def can_read(self):
if not self._reader:
raise ConnectionError("Socket closed on remote end")
if self._next_response is False:
self._next_response = self._reader.gets()
return self._next_response is not False
def write(self, data, try_left=None):
if try_left is None:
try_left = self.try_left
if not self._stream:
self.connect()
if not self._stream:
raise ConnectionError('Tried to write to non-existent connection')
if try_left > 0:
try:
self._stream.write(data)
except IOError:
self.disconnect()
self.write(data, try_left - 1)
else:
raise ConnectionError('Tried to write to non-existent connection')
def _read_from_socket(self, length=None):
socket_read_size = self.socket_read_size
buf = self._buffer
buf.seek(self.bytes_written)
marker = 0
try:
while True:
data = recv(self._sock, socket_read_size)
# an empty string indicates the server shutdown the socket
if isinstance(data, bytes) and len(data) == 0:
raise socket.error(SERVER_CLOSED_CONNECTION_ERROR)
buf.write(data)
data_length = len(data)
self.bytes_written += data_length
marker += data_length
if length is not None and length > marker:
continue
break
except socket.timeout:
raise TimeoutError("Timeout reading from socket")
except socket.error:
e = sys.exc_info()[1]
raise ConnectionError("Error while reading from socket: %s" %
(e.args, ))
def connect (self):
ftp = self.ftp = ftplib.FTP()
try:
ftp.connect(self.host)
except socket.error:
raise ConnectionError("Connecting to FTP server failed")
try:
ftp.login(self.username, self.password)
except ftplib.error_perm:
raise ConnectionError("Authentication failed")
if self.root:
if not self.root.startswith("/"):
self.root = "/" + self.root
self.cd(self.root)
self.cdRoot()
def can_read(self):
if not self._reader:
raise ConnectionError(SERVER_CLOSED_CONNECTION_ERROR)
if self._next_response is False:
self._next_response = self._reader.gets()
return self._next_response is not False
def get_connection(self, command_name, *keys, **options):
"""
Get a connection, blocking for ``self.timeout`` until a connection
is available from the pool.
If the connection returned is ``None`` then creates a new connection.
Because we use a last-in first-out queue, the existing connections
(having been returned to the pool after the initial ``None`` values
were added) will be returned before ``None`` values. This means we only
create new connections when we need to, i.e.: the actual number of
connections will only increase in response to demand.
"""
# Make sure we haven't changed process.
self._checkpid()
# Try and get a connection from the pool. If one isn't available within
# self.timeout then raise a ``ConnectionError``.
connection = None
try:
connection = self.pool.get(block=True, timeout=self.timeout)
except Empty:
# Note that this is not caught by the redis client and will be
# raised unless handled by application code. If you want never to
raise ConnectionError("No connection available.")
# If the ``connection`` is actually ``None`` then that's a cue to make
# a new connection to add to the pool.
if connection is None:
connection = self.make_connection()
return connection
def read(self, length, callback):
try:
if not self._stream:
self.disconnect()
raise ConnectionError('Tried to read from non-existent connection')
self._stream.read_bytes(length, callback)
except IOError:
self.fire_event('on_disconnect')
def _get(self, relative_url, *args, **kwargs):
"""
"""
for arg in args:
relative_url += '/{}'.format(arg)
url = urljoin(self.base_url, relative_url)
logger.debug('Hitting url: %s with params: %s', url, kwargs)
try:
response = self.session.get(url, params=kwargs)
except requests.ConnectionError:
raise ConnectionError(
'Unable to connect to %s - are you sure the subdomain is correct?'
% self.base_url)
try:
response.raise_for_status()
except requests.HTTPError as e:
if response.status_code == 401:
raise ConnectionError(
'Unable to authenticate - are you sure the API key is correct?'
)
elif response.status_code == 404:
raise ConnectionError(
'No such resource %s - are you sure the endpoints were configured correctly?'
% relative_url)
else:
raise (e)
try:
response_json = response.json()
except ValueError:
logger.exception('Response was not valid JSON!')
return None
if args:
data = response_json
else:
rows = response_json['rowcount']
data = response_json['data']
assert len(data) == rows
return data
def readline(self, callback):
try:
if not self._stream:
self.disconnect()
raise ConnectionError('Tried to read from non-existent connection')
saved_callback = stack_context.wrap(callback)
self.read_callbacks.append(saved_callback)
self._stream.read_until('\r\n',
callback=partial(self.readline_callback, saved_callback))
except IOError:
self.fire_event('on_disconnect')
def connect(self):
"""Connect to the server
:raise ConnectionError: If socket cannot establish a connection
"""
try:
logger.info(u'Connecting %s:%d' % (self.host, self.port))
self.sock.connect((self.host, self.port))
except socket.error:
raise ConnectionError()
self.state = CONNECTED
def read(self, length, callback=None):
try:
if not self._stream:
self.disconnect()
raise ConnectionError('Tried to read from '
'non-existent connection')
self.read_callbacks.append(callback)
self._stream.read_bytes(length,
callback=partial(self.read_callback,
callback))
except IOError:
self.fire_event('on_disconnect')
def get_entry(self, refresh=False):
if not refresh and self._entry is not None:
return etree.tostring(self._entry)
resp = requests.get(self.edit_uri, auth=self.connection.auth)
if resp.status_code != 200:
raise ConnectionError('Atom entry could not be retrieved.')
entry_string = resp.content
self._entry = etree.XML(entry_string)
return entry_string
def connect(self):
if not self._stream:
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
sock.settimeout(self.timeout)
sock.connect((self.host, self.port))
self._stream = IOStream(sock, io_loop=self._io_loop)
self._stream.set_close_callback(self.on_stream_close)
self.connected()
except socket.error, e:
raise ConnectionError(str(e))
self.fire_event('on_connect')
def on_connect(self):
"Initialize the connection, authenticate and select a database"
self._parser.on_connect(self)
# if a password is specified, authenticate
if self.password:
self.send_command('AUTH', self.password)
if nativestr(self.read_response()) != 'OK':
raise AuthenticationError('Invalid Password')
# if a database is specified, switch to it
if self.db:
self.send_command('SELECT', self.db)
if nativestr(self.read_response()) != 'OK':
raise ConnectionError('Invalid Database')
def get_contents(self, refresh=False):
if not refresh and self._contents_json:
return self._contents_json
content_uri = 'https://{0}/api/dataverses/{1}/contents'.format(
self.connection.host, self.alias
)
resp = requests.get(
content_uri,
params={'key': self.connection.token}
)
if resp.status_code != 200:
raise ConnectionError('Atom entry could not be retrieved.')
self._contents_json = resp.json()['data']
return self._contents_json
def read_response(self):
response = self._buffer.readline()
if not response:
raise ConnectionError("Socket closed on remote end")
byte, response = byte_to_chr(response[0]), response[1:]
if byte not in ('-', '+', ':', '$', '*'):
raise InvalidResponse("Protocol Error: %s, %s" %
(str(byte), str(response)))
# server returned an error
if byte == '-':
response = nativestr(response)
error = self.parse_error(response)
# if the error is a ConnectionError, raise immediately so the user
# is notified
if isinstance(error, ConnectionError):
raise error
# otherwise, we're dealing with a ResponseError that might belong
# inside a pipeline response. the connection's read_response()
# and/or the pipeline's execute() will raise this error if
# necessary, so just return the exception instance here.
return error
# single value
elif byte == '+':
pass
# int value
elif byte == ':':
response = long(response)
# bulk response
elif byte == '$':
length = int(response)
if length == -1:
return None
response = self._buffer.read(length)
# multi-bulk response
elif byte == '*':
length = int(response)
if length == -1:
return None
response = [self.read_response() for i in xrange(length)]
if isinstance(response, bytes) and self.encoding:
response = response.decode(self.encoding)
return response
def send_packed_command(self, command):
"Send an already packed command to the Redis server"
if not self._sock:
self.connect()
try:
self._sock.sendall(command)
except socket.error:
e = sys.exc_info()[1]
self.disconnect()
if len(e.args) == 1:
_errno, errmsg = 'UNKNOWN', e.args[0]
else:
_errno, errmsg = e.args
raise ConnectionError("Error %s while writing to socket. %s." %
(_errno, errmsg))
except:
self.disconnect()
raise
def _read_from_socket(self, length=None):
chunksize = 8192
if length is None:
length = chunksize
buf = self._buffer
buf.seek(self.bytes_written)
marker = 0
try:
while length > marker:
data = self._sock.recv(chunksize)
buf.write(data)
self.bytes_written += len(data)
marker += chunksize
except (socket.error, socket.timeout):
e = sys.exc_info()[1]
raise ConnectionError("Error while reading from socket: %s" %
(e.args, ))
def execute(self, callback=None):
command_stack = self.command_stack
self.command_stack = []
if self.transactional:
command_stack = [CmdLine('MULTI')] \
+ command_stack \
+ [CmdLine('EXEC')]
request = format_pipeline_request(command_stack)
try:
self.connection.write(request)
except IOError:
self.command_stack = []
self.connection.disconnect()
raise ConnectionError("Socket closed on remote end")
except Exception, e:
self.command_stack = []
self.connection.disconnect()
raise e
def connect(self):
"Connects to the Redis server if not already connected"
if self._sock:
return
try:
sock = self._connect()
except socket.error:
e = sys.exc_info()[1]
raise ConnectionError(self._error_message(e))
self._sock = sock
try:
self.on_connect()
except RedisError:
# clean up after any error in on_connect
self.disconnect()
raise
# run any user callbacks. right now the only internal callback
# is for pubsub channel/pattern resubscription
for callback in self._connect_callbacks:
callback(self)
def error_wrapper(e):
if isinstance(e, GeneratorExit):
return ConnectionError('Connection lost')
else:
return e
def on_disconnect(self):
if self.subscribed:
self.subscribed = False
raise ConnectionError("Socket closed on remote end")
args.reverse()
args.extend(bits)
args.reverse()
self.connection.write(self.format_command(*args, **kwargs))
except Exception, e:
self.connection.disconnect()
raise e
if (cmd in PUB_SUB_COMMANDS) or (self.subscribed and cmd == 'PUBLISH'):
result = True
else:
with self.connection as c:
result = None
data = yield gen.Task(c.readline)
if not data:
raise ConnectionError('no data received')
else:
resp = yield gen.Task(self.process_data, data, cmd_line)
result = self.format_reply(cmd_line, resp)
if callback:
callback(result)
@gen.engine
def process_data(self, data, cmd_line, callback=None):
data = data[:-2] # strip \r\n
if data == '$-1':
response = None
elif data == '*0' or data == '*-1':
response = []
else:
