class RobotConnection(object):
def __init__(self, user, passwd):
self.user = user
self.passwd = passwd
self.conn = ValidatedHTTPSConnection(ROBOT_HOST)
self.logger = logging.getLogger("Robot of {0}".format(user))
# Provide this as a way to easily add unsupported API features.
self.scraper = RobotWebInterface(user, passwd)
def _request(self, method, path, data, headers, retry=1):
self.conn.request(method.upper(), path, data, headers)
try:
return self.conn.getresponse()
except BadStatusLine:
# XXX: Sometimes, the API server seems to have a problem with
# keepalives.
if retry <= 0:
raise
self.conn.close()
self.conn.connect()
return self._request(method, path, data, headers, retry - 1)
def request(self, method, path, data=None, allow_empty=False):
if data is not None:
data = urlencode(data)
auth = 'Basic {0}'.format(b64encode(
"{0}:{1}".format(self.user, self.passwd).encode('ascii')
).decode('ascii'))
headers = {'Authorization': auth}
if data is not None:
headers['Content-Type'] = 'application/x-www-form-urlencoded'
self.logger.debug("Sending %s request to Robot at %s with data %r.",
method, path, data)
response = self._request(method, path, data, headers)
raw_data = response.read().decode('utf-8')
if len(raw_data) == 0 and not allow_empty:
msg = "Empty response, status {0}."
raise RobotError(msg.format(response.status), response.status)
elif not allow_empty:
try:
data = json.loads(raw_data)
except ValueError:
msg = "Response is not JSON (status {0}): {1}"
raise RobotError(msg.format(response.status, repr(raw_data)))
else:
data = None
self.logger.debug(
"Got response from Robot with status %d and data %r.",
response.status, data
)
if 200 <= response.status < 300:
return data
else:
error = data.get('error', None)
if error is None:
raise RobotError("Unknown error: {0}".format(data),
response.status)
else:
err = "{0} - {1}".format(error['status'], error['message'])
missing = error.get('missing', [])
invalid = error.get('invalid', [])
fields = []
if missing is not None:
fields += missing
if invalid is not None:
fields += invalid
if len(fields) > 0:
err += ", fields: {0}".format(', '.join(fields))
raise RobotError(err, response.status)
get = lambda s, p: s.request('GET', p)
post = lambda s, p, d: s.request('POST', p, d)
put = lambda s, p, d: s.request('PUT', p, d)
delete = lambda s, p, d=None: s.request('DELETE', p, d, allow_empty=True)
class RobotConnection(object):
def __init__(self, user, passwd):
self.user = user
self.passwd = passwd
self.conn = ValidatedHTTPSConnection(ROBOT_HOST)
self.logger = logging.getLogger("Robot of {0}".format(user))
# Provide this as a way to easily add unsupported API features.
self.scraper = RobotWebInterface(user, passwd)
def _request(self, method, path, data, headers, retry=1):
self.conn.request(method.upper(), path, data, headers)
try:
return self.conn.getresponse()
except BadStatusLine:
# XXX: Sometimes, the API server seems to have a problem with
# keepalives.
if retry <= 0:
raise
self.conn.close()
self.conn.connect()
return self._request(method, path, data, headers, retry - 1)
def _encode_phpargs(self, node, path=[]):
"""
Encode the given 'node' in a way PHP recognizes.
See https://php.net/manual/function.http-build-query.php for a
description of the format.
>>> robot = RobotConnection(None, None)
>>> enc = lambda arg: sorted(robot._encode_phpargs(arg).items())
>>> enc({'foo': [1, 2, 3]})
[('foo[0]', 1), ('foo[1]', 2), ('foo[2]', 3)]
>>> enc(['a', 'b', 'c'])
[('0', 'a'), ('1', 'b'), ('2', 'c')]
>>> enc({'a': {'b': [1, 2, 3], 'c': 'd'}})
[('a[b][0]', 1), ('a[b][1]', 2), ('a[b][2]', 3), ('a[c]', 'd')]
>>> enc({})
[]
>>> enc({'a': {'b': {'c': {}}}})
[]
>>> enc({'a': [1, 2, 3], 'b': {'c': 4}})
[('a[0]', 1), ('a[1]', 2), ('a[2]', 3), ('b[c]', 4)]
"""
if isinstance(node, list):
enum = enumerate(node)
elif isinstance(node, dict):
enum = node.items()
elif len(path) == 0:
return node
else:
# TODO: Implement escaping of keys.
flatkey = map(lambda x: '[' + str(x) + ']', path[1:])
return {str(path[0]) + ''.join(flatkey): node}
encoded = [self._encode_phpargs(v, path + [k]) for k, v in enum]
return functools.reduce(lambda a, b: a.update(b) or a, encoded, {})
def request(self, method, path, data=None, allow_empty=False):
if data is not None:
data = urlencode(self._encode_phpargs(data))
auth = 'Basic {0}'.format(b64encode(
"{0}:{1}".format(self.user, self.passwd).encode('ascii')
).decode('ascii'))
headers = {'Authorization': auth}
if data is not None:
headers['Content-Type'] = 'application/x-www-form-urlencoded'
self.logger.debug("Sending %s request to Robot at %s with data %r.",
method, path, data)
response = self._request(method, path, data, headers)
raw_data = response.read().decode('utf-8')
if len(raw_data) == 0 and not allow_empty:
msg = "Empty response, status {0}."
raise RobotError(msg.format(response.status), response.status)
elif not allow_empty:
try:
data = json.loads(raw_data)
except ValueError:
msg = "Response is not JSON (status {0}): {1}"
raise RobotError(msg.format(response.status, repr(raw_data)))
else:
data = None
self.logger.debug(
"Got response from Robot with status %d and data %r.",
response.status, data
)
if 200 <= response.status < 300:
return data
else:
error = data.get('error', None)
if error is None:
raise RobotError("Unknown error: {0}".format(data),
response.status)
else:
err = "{0} - {1}".format(error['status'], error['message'])
missing = error.get('missing', [])
invalid = error.get('invalid', [])
fields = []
if missing is not None:
fields += missing
if invalid is not None:
fields += invalid
if len(fields) > 0:
err += ", fields: {0}".format(', '.join(fields))
raise RobotError(err, response.status)
def get(self, path):
return self.request('GET', path)
def post(self, path, data):
return self.request('POST', path, data)
def put(self, path, data):
return self.request('PUT', path, data)
def delete(self, path, data=None):
return self.request('DELETE', path, data, allow_empty=True)
class RobotConnection(object):
def __init__(self, user, passwd):
self.user = user
self.passwd = passwd
self.conn = ValidatedHTTPSConnection(ROBOT_HOST)
self.logger = logging.getLogger("Robot of {0}".format(user))
# Provide this as a way to easily add unsupported API features.
self.scraper = RobotWebInterface(user, passwd)
def _request(self, method, path, data, headers, retry=1):
self.conn.request(method.upper(), path, data, headers)
try:
return self.conn.getresponse()
except BadStatusLine:
# XXX: Sometimes, the API server seems to have a problem with
# keepalives.
if retry <= 0:
raise
self.conn.close()
self.conn.connect()
return self._request(method, path, data, headers, retry - 1)
def request(self, method, path, data=None, allow_empty=False):
if data is not None:
data = urlencode(data)
auth = 'Basic {0}'.format(
b64encode("{0}:{1}".format(
self.user, self.passwd).encode('ascii')).decode('ascii'))
headers = {'Authorization': auth}
if data is not None:
headers['Content-Type'] = 'application/x-www-form-urlencoded'
self.logger.debug("Sending %s request to Robot at %s with data %r.",
method, path, data)
response = self._request(method, path, data, headers)
raw_data = response.read().decode('utf-8')
if len(raw_data) == 0 and not allow_empty:
msg = "Empty response, status {0}."
raise RobotError(msg.format(response.status), response.status)
elif not allow_empty:
try:
data = json.loads(raw_data)
except ValueError:
msg = "Response is not JSON (status {0}): {1}"
raise RobotError(msg.format(response.status, repr(raw_data)))
else:
data = None
self.logger.debug(
"Got response from Robot with status %d and data %r.",
response.status, data)
if 200 <= response.status < 300:
return data
else:
error = data.get('error', None)
if error is None:
raise RobotError("Unknown error: {0}".format(data),
response.status)
else:
err = "{0} - {1}".format(error['status'], error['message'])
missing = error.get('missing', [])
invalid = error.get('invalid', [])
fields = []
if missing is not None:
fields += missing
if invalid is not None:
fields += invalid
if len(fields) > 0:
err += ", fields: {0}".format(', '.join(fields))
raise RobotError(err, response.status)
def get(self, path):
return self.request('GET', path)
def post(self, path, data):
return self.request('POST', path, data)
def put(self, path, data):
return self.request('PUT', path, data)
def delete(self, path, data=None):
return self.request('DELETE', path, data, allow_empty=True)
class RobotConnection(object):
def __init__(self, user, passwd):
self.user = user
self.passwd = passwd
self.conn = ValidatedHTTPSConnection(ROBOT_HOST)
self.logger = logging.getLogger("Robot of {0}".format(user))
# Provide this as a way to easily add unsupported API features.
self.scraper = RobotWebInterface(user, passwd)
def _request(self, method, path, data, headers, retry=1):
self.conn.request(method.upper(), path, data, headers)
try:
return self.conn.getresponse()
except BadStatusLine:
# XXX: Sometimes, the API server seems to have a problem with
# keepalives.
if retry <= 0:
raise
self.conn.close()
self.conn.connect()
return self._request(method, path, data, headers, retry - 1)
def _encode_phpargs(self, node, path=[]):
"""
Encode the given 'node' in a way PHP recognizes.
See https://php.net/manual/function.http-build-query.php for a
description of the format.
>>> robot = RobotConnection(None, None)
>>> enc = lambda arg: sorted(robot._encode_phpargs(arg).items())
>>> enc({'foo': [1, 2, 3]})
[('foo[0]', 1), ('foo[1]', 2), ('foo[2]', 3)]
>>> enc(['a', 'b', 'c'])
[('0', 'a'), ('1', 'b'), ('2', 'c')]
>>> enc({'a': {'b': [1, 2, 3], 'c': 'd'}})
[('a[b][0]', 1), ('a[b][1]', 2), ('a[b][2]', 3), ('a[c]', 'd')]
>>> enc({})
[]
>>> enc({'a': {'b': {'c': {}}}})
[]
>>> enc({'a': [1, 2, 3], 'b': {'c': 4}})
[('a[0]', 1), ('a[1]', 2), ('a[2]', 3), ('b[c]', 4)]
"""
if isinstance(node, list):
enum = enumerate(node)
elif isinstance(node, dict):
enum = node.items()
elif len(path) == 0:
return node
else:
# TODO: Implement escaping of keys.
flatkey = map(lambda x: '[' + str(x) + ']', path[1:])
return {str(path[0]) + ''.join(flatkey): node}
encoded = [self._encode_phpargs(v, path + [k]) for k, v in enum]
return functools.reduce(lambda a, b: a.update(b) or a, encoded, {})
def request(self, method, path, data=None, allow_empty=False):
if data is not None:
data = urlencode(self._encode_phpargs(data))
auth = 'Basic {0}'.format(
b64encode("{0}:{1}".format(
self.user, self.passwd).encode('ascii')).decode('ascii'))
headers = {'Authorization': auth}
if data is not None:
headers['Content-Type'] = 'application/x-www-form-urlencoded'
self.logger.debug("Sending %s request to Robot at %s with data %r.",
method, path, data)
response = self._request(method, path, data, headers)
raw_data = response.read().decode('utf-8')
if len(raw_data) == 0 and not allow_empty:
msg = "Empty response, status {0}."
raise RobotError(msg.format(response.status), response.status)
elif not allow_empty:
try:
data = json.loads(raw_data)
except ValueError:
msg = "Response is not JSON (status {0}): {1}"
raise RobotError(msg.format(response.status, repr(raw_data)))
else:
data = None
self.logger.debug(
"Got response from Robot with status %d and data %r.",
response.status, data)
if 200 <= response.status < 300:
return data
else:
error = data.get('error', None)
if error is None:
raise RobotError("Unknown error: {0}".format(data),
response.status)
else:
err = "{0} - {1}".format(error['status'], error['message'])
missing = error.get('missing', [])
invalid = error.get('invalid', [])
fields = []
if missing is not None:
fields += missing
if invalid is not None:
fields += invalid
if len(fields) > 0:
err += ", fields: {0}".format(', '.join(fields))
raise RobotError(err, response.status)
def get(self, path):
return self.request('GET', path)
def post(self, path, data):
return self.request('POST', path, data)
def put(self, path, data):
return self.request('PUT', path, data)
def delete(self, path, data=None):
return self.request('DELETE', path, data, allow_empty=True)
