class RobotConnection(object): def __init__(self, user, passwd): self.user = user self.passwd = passwd self.conn = ValidatedHTTPSConnection(ROBOT_HOST) self.logger = logging.getLogger("Robot of {0}".format(user)) # Provide this as a way to easily add unsupported API features. self.scraper = RobotWebInterface(user, passwd) def _request(self, method, path, data, headers, retry=1): self.conn.request(method.upper(), path, data, headers) try: return self.conn.getresponse() except BadStatusLine: # XXX: Sometimes, the API server seems to have a problem with # keepalives. if retry <= 0: raise self.conn.close() self.conn.connect() return self._request(method, path, data, headers, retry - 1) def request(self, method, path, data=None, allow_empty=False): if data is not None: data = urlencode(data) auth = 'Basic {0}'.format(b64encode( "{0}:{1}".format(self.user, self.passwd).encode('ascii') ).decode('ascii')) headers = {'Authorization': auth} if data is not None: headers['Content-Type'] = 'application/x-www-form-urlencoded' self.logger.debug("Sending %s request to Robot at %s with data %r.", method, path, data) response = self._request(method, path, data, headers) raw_data = response.read().decode('utf-8') if len(raw_data) == 0 and not allow_empty: msg = "Empty response, status {0}." raise RobotError(msg.format(response.status), response.status) elif not allow_empty: try: data = json.loads(raw_data) except ValueError: msg = "Response is not JSON (status {0}): {1}" raise RobotError(msg.format(response.status, repr(raw_data))) else: data = None self.logger.debug( "Got response from Robot with status %d and data %r.", response.status, data ) if 200 <= response.status < 300: return data else: error = data.get('error', None) if error is None: raise RobotError("Unknown error: {0}".format(data), response.status) else: err = "{0} - {1}".format(error['status'], error['message']) missing = error.get('missing', []) invalid = error.get('invalid', []) fields = [] if missing is not None: fields += missing if invalid is not None: fields += invalid if len(fields) > 0: err += ", fields: {0}".format(', '.join(fields)) raise RobotError(err, response.status) get = lambda s, p: s.request('GET', p) post = lambda s, p, d: s.request('POST', p, d) put = lambda s, p, d: s.request('PUT', p, d) delete = lambda s, p, d=None: s.request('DELETE', p, d, allow_empty=True)
class RobotConnection(object): def __init__(self, user, passwd): self.user = user self.passwd = passwd self.conn = ValidatedHTTPSConnection(ROBOT_HOST) self.logger = logging.getLogger("Robot of {0}".format(user)) # Provide this as a way to easily add unsupported API features. self.scraper = RobotWebInterface(user, passwd) def _request(self, method, path, data, headers, retry=1): self.conn.request(method.upper(), path, data, headers) try: return self.conn.getresponse() except BadStatusLine: # XXX: Sometimes, the API server seems to have a problem with # keepalives. if retry <= 0: raise self.conn.close() self.conn.connect() return self._request(method, path, data, headers, retry - 1) def _encode_phpargs(self, node, path=[]): """ Encode the given 'node' in a way PHP recognizes. See https://php.net/manual/function.http-build-query.php for a description of the format. >>> robot = RobotConnection(None, None) >>> enc = lambda arg: sorted(robot._encode_phpargs(arg).items()) >>> enc({'foo': [1, 2, 3]}) [('foo[0]', 1), ('foo[1]', 2), ('foo[2]', 3)] >>> enc(['a', 'b', 'c']) [('0', 'a'), ('1', 'b'), ('2', 'c')] >>> enc({'a': {'b': [1, 2, 3], 'c': 'd'}}) [('a[b][0]', 1), ('a[b][1]', 2), ('a[b][2]', 3), ('a[c]', 'd')] >>> enc({}) [] >>> enc({'a': {'b': {'c': {}}}}) [] >>> enc({'a': [1, 2, 3], 'b': {'c': 4}}) [('a[0]', 1), ('a[1]', 2), ('a[2]', 3), ('b[c]', 4)] """ if isinstance(node, list): enum = enumerate(node) elif isinstance(node, dict): enum = node.items() elif len(path) == 0: return node else: # TODO: Implement escaping of keys. flatkey = map(lambda x: '[' + str(x) + ']', path[1:]) return {str(path[0]) + ''.join(flatkey): node} encoded = [self._encode_phpargs(v, path + [k]) for k, v in enum] return functools.reduce(lambda a, b: a.update(b) or a, encoded, {}) def request(self, method, path, data=None, allow_empty=False): if data is not None: data = urlencode(self._encode_phpargs(data)) auth = 'Basic {0}'.format(b64encode( "{0}:{1}".format(self.user, self.passwd).encode('ascii') ).decode('ascii')) headers = {'Authorization': auth} if data is not None: headers['Content-Type'] = 'application/x-www-form-urlencoded' self.logger.debug("Sending %s request to Robot at %s with data %r.", method, path, data) response = self._request(method, path, data, headers) raw_data = response.read().decode('utf-8') if len(raw_data) == 0 and not allow_empty: msg = "Empty response, status {0}." raise RobotError(msg.format(response.status), response.status) elif not allow_empty: try: data = json.loads(raw_data) except ValueError: msg = "Response is not JSON (status {0}): {1}" raise RobotError(msg.format(response.status, repr(raw_data))) else: data = None self.logger.debug( "Got response from Robot with status %d and data %r.", response.status, data ) if 200 <= response.status < 300: return data else: error = data.get('error', None) if error is None: raise RobotError("Unknown error: {0}".format(data), response.status) else: err = "{0} - {1}".format(error['status'], error['message']) missing = error.get('missing', []) invalid = error.get('invalid', []) fields = [] if missing is not None: fields += missing if invalid is not None: fields += invalid if len(fields) > 0: err += ", fields: {0}".format(', '.join(fields)) raise RobotError(err, response.status) def get(self, path): return self.request('GET', path) def post(self, path, data): return self.request('POST', path, data) def put(self, path, data): return self.request('PUT', path, data) def delete(self, path, data=None): return self.request('DELETE', path, data, allow_empty=True)
class RobotConnection(object): def __init__(self, user, passwd): self.user = user self.passwd = passwd self.conn = ValidatedHTTPSConnection(ROBOT_HOST) self.logger = logging.getLogger("Robot of {0}".format(user)) # Provide this as a way to easily add unsupported API features. self.scraper = RobotWebInterface(user, passwd) def _request(self, method, path, data, headers, retry=1): self.conn.request(method.upper(), path, data, headers) try: return self.conn.getresponse() except BadStatusLine: # XXX: Sometimes, the API server seems to have a problem with # keepalives. if retry <= 0: raise self.conn.close() self.conn.connect() return self._request(method, path, data, headers, retry - 1) def request(self, method, path, data=None, allow_empty=False): if data is not None: data = urlencode(data) auth = 'Basic {0}'.format( b64encode("{0}:{1}".format( self.user, self.passwd).encode('ascii')).decode('ascii')) headers = {'Authorization': auth} if data is not None: headers['Content-Type'] = 'application/x-www-form-urlencoded' self.logger.debug("Sending %s request to Robot at %s with data %r.", method, path, data) response = self._request(method, path, data, headers) raw_data = response.read().decode('utf-8') if len(raw_data) == 0 and not allow_empty: msg = "Empty response, status {0}." raise RobotError(msg.format(response.status), response.status) elif not allow_empty: try: data = json.loads(raw_data) except ValueError: msg = "Response is not JSON (status {0}): {1}" raise RobotError(msg.format(response.status, repr(raw_data))) else: data = None self.logger.debug( "Got response from Robot with status %d and data %r.", response.status, data) if 200 <= response.status < 300: return data else: error = data.get('error', None) if error is None: raise RobotError("Unknown error: {0}".format(data), response.status) else: err = "{0} - {1}".format(error['status'], error['message']) missing = error.get('missing', []) invalid = error.get('invalid', []) fields = [] if missing is not None: fields += missing if invalid is not None: fields += invalid if len(fields) > 0: err += ", fields: {0}".format(', '.join(fields)) raise RobotError(err, response.status) def get(self, path): return self.request('GET', path) def post(self, path, data): return self.request('POST', path, data) def put(self, path, data): return self.request('PUT', path, data) def delete(self, path, data=None): return self.request('DELETE', path, data, allow_empty=True)
class RobotConnection(object): def __init__(self, user, passwd): self.user = user self.passwd = passwd self.conn = ValidatedHTTPSConnection(ROBOT_HOST) self.logger = logging.getLogger("Robot of {0}".format(user)) # Provide this as a way to easily add unsupported API features. self.scraper = RobotWebInterface(user, passwd) def _request(self, method, path, data, headers, retry=1): self.conn.request(method.upper(), path, data, headers) try: return self.conn.getresponse() except BadStatusLine: # XXX: Sometimes, the API server seems to have a problem with # keepalives. if retry <= 0: raise self.conn.close() self.conn.connect() return self._request(method, path, data, headers, retry - 1) def _encode_phpargs(self, node, path=[]): """ Encode the given 'node' in a way PHP recognizes. See https://php.net/manual/function.http-build-query.php for a description of the format. >>> robot = RobotConnection(None, None) >>> enc = lambda arg: sorted(robot._encode_phpargs(arg).items()) >>> enc({'foo': [1, 2, 3]}) [('foo[0]', 1), ('foo[1]', 2), ('foo[2]', 3)] >>> enc(['a', 'b', 'c']) [('0', 'a'), ('1', 'b'), ('2', 'c')] >>> enc({'a': {'b': [1, 2, 3], 'c': 'd'}}) [('a[b][0]', 1), ('a[b][1]', 2), ('a[b][2]', 3), ('a[c]', 'd')] >>> enc({}) [] >>> enc({'a': {'b': {'c': {}}}}) [] >>> enc({'a': [1, 2, 3], 'b': {'c': 4}}) [('a[0]', 1), ('a[1]', 2), ('a[2]', 3), ('b[c]', 4)] """ if isinstance(node, list): enum = enumerate(node) elif isinstance(node, dict): enum = node.items() elif len(path) == 0: return node else: # TODO: Implement escaping of keys. flatkey = map(lambda x: '[' + str(x) + ']', path[1:]) return {str(path[0]) + ''.join(flatkey): node} encoded = [self._encode_phpargs(v, path + [k]) for k, v in enum] return functools.reduce(lambda a, b: a.update(b) or a, encoded, {}) def request(self, method, path, data=None, allow_empty=False): if data is not None: data = urlencode(self._encode_phpargs(data)) auth = 'Basic {0}'.format( b64encode("{0}:{1}".format( self.user, self.passwd).encode('ascii')).decode('ascii')) headers = {'Authorization': auth} if data is not None: headers['Content-Type'] = 'application/x-www-form-urlencoded' self.logger.debug("Sending %s request to Robot at %s with data %r.", method, path, data) response = self._request(method, path, data, headers) raw_data = response.read().decode('utf-8') if len(raw_data) == 0 and not allow_empty: msg = "Empty response, status {0}." raise RobotError(msg.format(response.status), response.status) elif not allow_empty: try: data = json.loads(raw_data) except ValueError: msg = "Response is not JSON (status {0}): {1}" raise RobotError(msg.format(response.status, repr(raw_data))) else: data = None self.logger.debug( "Got response from Robot with status %d and data %r.", response.status, data) if 200 <= response.status < 300: return data else: error = data.get('error', None) if error is None: raise RobotError("Unknown error: {0}".format(data), response.status) else: err = "{0} - {1}".format(error['status'], error['message']) missing = error.get('missing', []) invalid = error.get('invalid', []) fields = [] if missing is not None: fields += missing if invalid is not None: fields += invalid if len(fields) > 0: err += ", fields: {0}".format(', '.join(fields)) raise RobotError(err, response.status) def get(self, path): return self.request('GET', path) def post(self, path, data): return self.request('POST', path, data) def put(self, path, data): return self.request('PUT', path, data) def delete(self, path, data=None): return self.request('DELETE', path, data, allow_empty=True)