def __init__( self, host, root_path, proxy=None, verify_ssl=True, retry_event=None ): # retry_event should be an Event Object, use to cancel retry loops, if the event get's set the retry loop will throw the most recent Exception it ignored super().__init__() self.retry_event = retry_event if not host.startswith(('http:', 'https:')): raise ValueError('hostname must start with http(s):') if host[-1] == '/': raise ValueError('hostname must not end with "/"') self.proxy = proxy self.host = host logging.debug( 'cinp: new client host: "{0}", root_path: "{1}", via: "{2}"'. format(self.host, root_path, self.proxy)) self.uri = URI(root_path) self.opener = request.OpenerDirector() if self.proxy: # not doing 'is not None', so empty strings don't try and proxy # have a proxy option to take it from the envrionment vars self.opener.add_handler( request.ProxyHandler({ 'http': self.proxy, 'https': self.proxy })) else: self.opener.add_handler(request.ProxyHandler({})) self.opener.add_handler(request.HTTPHandler()) if hasattr(http.client, 'HTTPSConnection'): if not verify_ssl: self.opener.add_handler( request.HTTPSHandler( context=ssl._create_unverified_context())) else: self.opener.add_handler(request.HTTPSHandler()) self.opener.add_handler(request.UnknownHandler()) self.opener.addheaders = [ ('User-Agent', 'python CInP client {0}'.format(__CLIENT_VERSION__)), ('Accepts', 'application/json'), ('Accept-Charset', 'utf-8'), ('CInP-Version', __CINP_VERSION__) ]
def uploadFile(self, uri, filepath, filename=None, cb=None, timeout=30): """ filepath can be a string of the path name or a file object. If a file object either specify the filename or make sure your file object exposes the attribute 'name'. Also if file object, must be opened in binary mode, ie: 'rb' NOTE: this is not a CInP function, but a conviance function for uploading large files. """ uri_parser = URI('/') try: # TODO: There has to be a better way to validate this uri (namespace, model, action, id_list, _) = uri_parser.split(uri) except ValueError as e: raise InvalidRequest(str(e)) if action is not None or id_list is not None: raise InvalidRequest( 'file upload target can\'t be an action nor have ids') if isinstance(filepath, str): if filename is None: filename = os.path.basename(filepath) file_reader = _readerWrapper(open(filepath, 'rb'), cb) else: if filename is None: filename = os.path.basename(filepath.name) file_reader = _readerWrapper(filepath, cb) header_map = { 'Content-Disposition': 'inline: filename="{0}"'.format(filename), 'Content-Length': len(file_reader) } (http_code, data, _) = self._request('UPLOAD', uri_parser.build(namespace, model), data=file_reader, header_map=header_map, timeout=timeout) if http_code != 202: logging.warning( 'cinp: Unexpected HTTP Code "{0}" for File Upload'.format( http_code)) raise ResponseError( 'Unexpected HTTP Code "{0}" for File Upload'.format(http_code)) return data['uri']
def test_splituri_builduri(): # TODO: test invlid URIs, mabey remove some tests from client_test that are just checking the URI uri = URI( '/api/v1/' ) ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/' ) assert ns == [] assert model is None assert id_list is None assert action is None assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/' ns = None assert uri.build( ns, model, action, id_list ) == '/api/v1/' ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/ns/' ) assert ns == [ 'ns' ] assert model is None assert id_list is None assert action is None assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/' ns = 'ns' assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/' ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/ns/model' ) assert ns == [ 'ns' ] assert model == 'model' assert id_list is None assert action is None assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/model' id_list = [] assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/model' ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/ns/ns2/' ) assert ns == [ 'ns', 'ns2' ] assert model is None assert id_list is None assert action is None assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/ns2/' ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/ns/ns2/model' ) assert ns == [ 'ns', 'ns2' ] assert model == 'model' assert id_list is None assert action is None assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/ns2/model' ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/ns/model::' ) assert ns == [ 'ns' ] assert model == 'model' assert id_list == [ '' ] assert action is None assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/model::' id_list = '' assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/model::' ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/ns/model:ghj:' ) assert ns == [ 'ns' ] assert model == 'model' assert id_list == [ 'ghj' ] assert action is None assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/model:ghj:' id_list = 'ghj' assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/model:ghj:' ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/ns/model:ghj:dsf:sfe:' ) assert ns == [ 'ns' ] assert model == 'model' assert id_list == [ 'ghj', 'dsf', 'sfe' ] assert action is None assert multi is True assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/model:ghj:dsf:sfe:' ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/ns/model(action)' ) assert ns == [ 'ns' ] assert model == 'model' assert id_list is None assert action == 'action' assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/model(action)' ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/ns/model:sdf:(action)' ) assert ns == [ 'ns' ] assert model == 'model' assert id_list == [ 'sdf' ] assert action == 'action' assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/model:sdf:(action)' ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/ns/model:sdf:eed:(action)' ) assert ns == [ 'ns' ] assert model == 'model' assert id_list == [ 'sdf', 'eed' ] assert action == 'action' assert multi is True assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/model:sdf:eed:(action)' ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/', root_optional=True ) assert ns == [] assert model is None assert id_list is None assert action is None assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/' assert uri.build( ns, model, action, id_list, in_root=False ) == '/' with pytest.raises( ValueError ): ( ns, model, action, id_list, multi ) = uri.split( '/', root_optional=False ) ( ns, model, action, id_list, multi ) = uri.split( '/', root_optional=True ) assert ns == [] assert model is None assert id_list is None assert action is None assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/' assert uri.build( ns, model, action, id_list, in_root=False ) == '/' ( ns, model, action, id_list, multi ) = uri.split( '/api/v1/ns/', root_optional=True ) assert ns == [ 'ns' ] assert model is None assert id_list is None assert action is None assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/' assert uri.build( ns, model, action, id_list, in_root=False ) == '/ns/' with pytest.raises( ValueError ): ( ns, model, action, id_list, multi ) = uri.split( '/ns/', root_optional=False ) ( ns, model, action, id_list, multi ) = uri.split( '/ns/', root_optional=True ) assert ns == [ 'ns' ] assert model is None assert id_list is None assert action is None assert multi is False assert uri.build( ns, model, action, id_list ) == '/api/v1/ns/' assert uri.build( ns, model, action, id_list, in_root=False ) == '/ns/'
def test_extract_ids(): uri = URI( '/api/v1/' ) id_list = [ '/api/v1/ns/model:sdf:', '/api/v1/ns/model:234:', '/api/v1/ns/model:rfv:' ] assert uri.extractIds( id_list ) == [ 'sdf', '234', 'rfv' ] id_list = [ '/api/v1/ns/model:sdf:', '/api/v1/ns/model:234:www:', '/api/v1/ns/model:rfv:' ] assert uri.extractIds( id_list ) == [ 'sdf', '234', 'www', 'rfv' ] id_list = [ '/api/v1/ns/model:234:www:' ] assert uri.extractIds( id_list ) == [ '234', 'www' ] id_list = [ '/api/v1/ns/model:sdf:', '/api/v1/ns/model:234:www', '/api/v1/ns/model:rfv:' ] with pytest.raises( ValueError ): uri.extractIds( id_list ) id_list = [ '/api/v1/ns/model:sdf' ] with pytest.raises( ValueError ): uri.extractIds( id_list ) id_list = [ '/api/v1/ns/model' ] with pytest.raises( ValueError ): uri.extractIds( id_list ) id_list = [ '/api/v1/ns/model:sdf:', '/api/v1/ns/model', '/api/v1/ns/model:rfv:' ] with pytest.raises( ValueError ): uri.extractIds( id_list )
def test_not_allowed_verbs(): server = Server( root_path='/api/', root_version='0.0', debug=True ) ns1 = Namespace( name='ns1', version='0.1', converter=Converter( URI( '/api/' ) ) ) ns1.checkAuth = lambda user, verb, id_list: True field_list = [] field_list.append( Field( name='field1', type='String', length=50 ) ) model1 = Model( name='model1', field_list=field_list, not_allowed_verb_list=[], transaction_class=TestTransaction ) model2 = Model( name='model2', field_list=field_list, not_allowed_verb_list=[ 'GET', 'LIST', 'CALL', 'CREATE', 'UPDATE', 'DELETE', 'DESCRIBE' ], transaction_class=TestTransaction ) model1.checkAuth = lambda user, verb, id_list: True model2.checkAuth = lambda user, verb, id_list: True action1 = Action( name='act', return_paramater=Paramater( type='String' ), func=fake_func ) action2 = Action( name='act', return_paramater=Paramater( type='String' ), func=fake_func ) action1.checkAuth = lambda user, verb, id_list: True action2.checkAuth = lambda user, verb, id_list: True model1.addAction( action1 ) model2.addAction( action2 ) ns1.addElement( model1 ) ns1.addElement( model2 ) server.registerNamespace( '/', ns1 ) with pytest.raises( ValueError ): Model( name='modelX', field_list=[], not_allowed_verb_list=[ 'OPTIONS' ], transaction_class=TestTransaction ) with pytest.raises( ValueError ): Model( name='modelX', field_list=[], not_allowed_verb_list=[ 'ASDF' ], transaction_class=TestTransaction ) req = Request( 'OPTIONS', '/api/ns1/model1', { 'CINP-VERSION': __CINP_VERSION__ } ) res = server.handle( req ) assert res.http_code == 200 req = Request( 'OPTIONS', '/api/ns1/model2', { 'CINP-VERSION': __CINP_VERSION__ } ) res = server.handle( req ) assert res.http_code == 200 req = Request( 'DESCRIBE', '/api/ns1/model1', { 'CINP-VERSION': __CINP_VERSION__ } ) res = server.handle( req ) assert res.http_code == 200 req = Request( 'DESCRIBE', '/api/ns1/model2', { 'CINP-VERSION': __CINP_VERSION__ } ) res = server.handle( req ) assert res.http_code == 403 req = Request( 'GET', '/api/ns1/model1:asd:', { 'CINP-VERSION': __CINP_VERSION__ } ) res = server.handle( req ) assert res.http_code == 200 req = Request( 'GET', '/api/ns1/model2:asd:', { 'CINP-VERSION': __CINP_VERSION__ } ) res = server.handle( req ) assert res.http_code == 403 req = Request( 'LIST', '/api/ns1/model1', { 'CINP-VERSION': __CINP_VERSION__ } ) res = server.handle( req ) assert res.http_code == 200 req = Request( 'LIST', '/api/ns1/model2', { 'CINP-VERSION': __CINP_VERSION__ } ) res = server.handle( req ) assert res.http_code == 403 req = Request( 'CREATE', '/api/ns1/model1', { 'CINP-VERSION': __CINP_VERSION__ } ) req.data = { 'field1': 'stuff' } res = server.handle( req ) assert res.http_code == 201 req = Request( 'CREATE', '/api/ns1/model2', { 'CINP-VERSION': __CINP_VERSION__ } ) req.data = { 'field1': 'stuff' } res = server.handle( req ) assert res.http_code == 403 req = Request( 'UPDATE', '/api/ns1/model1:sdf:', { 'CINP-VERSION': __CINP_VERSION__ } ) req.data = { 'field1': 'stuff' } res = server.handle( req ) assert res.http_code == 200 req = Request( 'UPDATE', '/api/ns1/model2:sdf:', { 'CINP-VERSION': __CINP_VERSION__ } ) req.data = { 'field1': 'stuff' } res = server.handle( req ) assert res.http_code == 403 req = Request( 'DELETE', '/api/ns1/model1:asd:', { 'CINP-VERSION': __CINP_VERSION__ } ) res = server.handle( req ) assert res.http_code == 200 req = Request( 'DELETE', '/api/ns1/model2:asd:', { 'CINP-VERSION': __CINP_VERSION__ } ) res = server.handle( req ) assert res.http_code == 403 req = Request( 'CALL', '/api/ns1/model1(act)', { 'CINP-VERSION': __CINP_VERSION__ } ) res = server.handle( req ) assert res.http_code == 200 req = Request( 'CALL', '/api/ns1/model2(act)', { 'CINP-VERSION': __CINP_VERSION__ } ) res = server.handle( req ) assert res.http_code == 403
def test_getElement(): uri = URI( root_path='/api/' ) root_ns = Namespace( name=None, version='0.0', root_path='/api/', converter=None ) ns2 = Namespace( name='ns2', version='0.1', converter=None ) ns3 = Namespace( name='ns3', version='0.2', converter=None ) ns2_2 = Namespace( name='ns2_2', version='0.1', converter=None ) root_ns.addElement( ns2 ) root_ns.addElement( ns3 ) ns2.addElement( ns2_2 ) mdl1 = Model( name='mdl1', field_list={}, transaction_class=TestTransaction ) mdl2 = Model( name='mdl2', field_list={}, transaction_class=TestTransaction ) root_ns.addElement( mdl1 ) ns2_2.addElement( mdl2 ) act1 = Action( name='act1', return_paramater=Paramater( type='String' ), paramater_list=[ Paramater( name='bob', type='Float' ) ], static=False, func=fake_func ) act2 = Action( name='act2', return_paramater=Paramater( type='String' ), paramater_list=[ Paramater( name='stuff', type='Boolean' ) ], func=fake_func ) mdl1.addAction( act1 ) mdl2.addAction( act2 ) assert root_ns.getElement( uri.split( '/api/', root_optional=True ) ) == root_ns assert root_ns.getElement( uri.split( '/api/ns2/', root_optional=True ) ) == ns2 assert root_ns.getElement( uri.split( '/api/ns3/', root_optional=True ) ) == ns3 assert root_ns.getElement( uri.split( '/api/ns2/ns2_2/', root_optional=True ) ) == ns2_2 assert root_ns.getElement( uri.split( '/', root_optional=True ) ) == root_ns assert root_ns.getElement( uri.split( '/ns2/', root_optional=True ) ) == ns2 assert root_ns.getElement( uri.split( '/ns3/', root_optional=True ) ) == ns3 assert root_ns.getElement( uri.split( '/ns2/ns2_2/', root_optional=True ) ) == ns2_2 assert root_ns.getElement( uri.split( '/api/mdl1', root_optional=True ) ) == mdl1 assert root_ns.getElement( uri.split( '/api/ns2/ns2_2/mdl2', root_optional=True ) ) == mdl2 assert root_ns.getElement( uri.split( '/mdl1', root_optional=True ) ) == mdl1 assert root_ns.getElement( uri.split( '/ns2/ns2_2/mdl2', root_optional=True ) ) == mdl2 assert root_ns.getElement( uri.split( '/api/mdl1(act1)', root_optional=True ) ) == act1 assert root_ns.getElement( uri.split( '/api/ns2/ns2_2/mdl2(act2)', root_optional=True ) ) == act2 assert root_ns.getElement( uri.split( '/mdl1(act1)', root_optional=True ) ) == act1 assert root_ns.getElement( uri.split( '/ns2/ns2_2/mdl2(act2)', root_optional=True ) ) == act2 with pytest.raises( ValueError ): root_ns.getElement( '/api/' ) assert root_ns.getElement( uri.split( '/api/nsX/' ) ) is None assert root_ns.getElement( uri.split( '/api/ns2/mdlX' ) ) is None assert root_ns.getElement( uri.split( '/api/mdl1(actX)' ) ) is None
class CInP(): def __init__( self, host, root_path, proxy=None, verify_ssl=True, retry_event=None ): # retry_event should be an Event Object, use to cancel retry loops, if the event get's set the retry loop will throw the most recent Exception it ignored super().__init__() self.retry_event = retry_event if not host.startswith(('http:', 'https:')): raise ValueError('hostname must start with http(s):') if host[-1] == '/': raise ValueError('hostname must not end with "/"') self.proxy = proxy self.host = host logging.debug( 'cinp: new client host: "{0}", root_path: "{1}", via: "{2}"'. format(self.host, root_path, self.proxy)) self.uri = URI(root_path) self.opener = request.OpenerDirector() if self.proxy: # not doing 'is not None', so empty strings don't try and proxy # have a proxy option to take it from the envrionment vars self.opener.add_handler( request.ProxyHandler({ 'http': self.proxy, 'https': self.proxy })) else: self.opener.add_handler(request.ProxyHandler({})) self.opener.add_handler(request.HTTPHandler()) if hasattr(http.client, 'HTTPSConnection'): if not verify_ssl: self.opener.add_handler( request.HTTPSHandler( context=ssl._create_unverified_context())) else: self.opener.add_handler(request.HTTPSHandler()) self.opener.add_handler(request.UnknownHandler()) self.opener.addheaders = [ ('User-Agent', 'python CInP client {0}'.format(__CLIENT_VERSION__)), ('Accepts', 'application/json'), ('Accept-Charset', 'utf-8'), ('CInP-Version', __CINP_VERSION__) ] def _checkRequest( self, verb, uri, data ): # TODO: also check if verb is allowed to have headers ( other than thoes provided by the opener ), also check to make sure they are valid heaaders logging.debug('cinp: check "{0}" to "{1}"'.format(verb, uri)) if verb not in ('GET', 'LIST', 'UPDATE', 'CREATE', 'DELETE', 'CALL', 'DESCRIBE'): raise InvalidRequest( 'Invalid Verb (HTTP Method) "{0}"'.format(verb)) if data is not None and not isinstance(data, dict): raise InvalidRequest('Data must be a dict') try: (_, model, action, id_list, _) = self.uri.split(uri) except ValueError as e: raise InvalidRequest(str(e)) if id_list == []: id_list = None if action is not None and verb not in ('CALL', 'DESCRIBE'): raise InvalidRequest( 'Invalid verb "{0}" for request with action'.format(verb)) if verb in ('CALL', ) and action is None: raise InvalidRequest('Verb "{0}" requires action'.format(verb)) if id_list is not None and verb not in ('GET', 'UPDATE', 'DELETE', 'CALL'): raise InvalidRequest( 'Invalid verb "{0}" for request with id'.format(verb)) if verb in ('GET', 'UPDATE', 'DELETE') and id_list is None: raise InvalidRequest('verb "{0}" requires id'.format(verb)) if data is not None and verb not in ('LIST', 'UPDATE', 'CREATE', 'CALL'): raise InvalidRequest( 'Invalid verb "{0}" for request with data'.format(verb)) if verb in ('UPDATE', 'CREATE') and data is None: raise InvalidRequest('Verb "{0}" requires data'.format(verb)) if verb in ('GET', 'LIST', 'UPDATE', 'CREATE', 'DELETE', 'CALL') and not model: raise InvalidRequest('Verb "{0}" requires model'.format(verb)) def _request(self, verb, uri, data=None, header_map=None, timeout=30, retry_count=0, retry_on=None): _retry_on = retry_on or DEFAULT_RETRY_ON last_exception = None for retry in range(0, retry_count + 1): if retry > 0: logging.debug( 'cinp: retry "{0}" of "{1}" for request to "{2}"'.format( retry, retry_count, uri)) if self.retry_event is not None: if self.retry_event.is_set(): raise last_exception self.retry_event.wait(_backOffDelay(retry)) else: time.sleep(_backOffDelay(retry)) try: return self.__request(verb, uri, data, header_map, timeout) except RetryableException as e: signature = '{0}:{1}'.format(e.exception.__class__.__name__, e.reason) if signature not in _retry_on: logging.debug( 'cinp: exception signature "{0}" not in the retry_on list' .format(signature)) raise e.exception logging.debug( 'cinp: got exception "{0}", ignoring...'.format(e)) last_exception = e.exception raise last_exception def __request(self, verb, uri, data=None, header_map=None, timeout=30): logging.debug('cinp: making "{0}" request to "{1}"'.format(verb, uri)) if header_map is None: header_map = {} if verb == 'UPLOAD': # not a CINP verb, just using it to bypass some checking here in __request header_map['Content-Type'] = 'application/octet-stream' if not hasattr(data, 'read'): raise InvalidRequest('data must be an readable stream') verb = 'POST' # not to be handled by CInP on the other end, but by a file upload handler elif verb == 'RAWGET': # not a CINP verb, just using it to bypass some checking here in __request verb = 'GET' else: header_map['Content-Type'] = 'application/json;charset=utf-8' self._checkRequest(verb, uri, data) if data is None: data = ''.encode('utf-8') else: data = json.dumps(data, default=JSONDefault).encode('utf-8') url = '{0}{1}'.format(self.host, uri) req = request.Request(url, data=data, headers=header_map, method=verb) try: resp = self.opener.open(req, timeout=timeout) except request.HTTPError as e: raise RetryableException(ResponseError( 'HTTPError "{0}"'.format(e)), reason=e.reason) except request.URLError as e: if isinstance(e.reason, socket.timeout): raise RetryableException( Timeout( 'Request Timeout after {0} seconds'.format(timeout))) raise RetryableException(ResponseError( 'URLError "{0}" for "{1}" via "{2}"'.format( e, url, self.proxy)), reason=e.reason) except socket.timeout: raise RetryableException( Timeout('Request Timeout after {0} seconds'.format(timeout))) except OSError as e: raise RetryableException(ResponseError( 'Socket Error "{0}"'.format(e)), reason=e.strerror) http_code = resp.code if http_code not in (200, 201, 202, 400, 401, 403, 404, 500): raise ResponseError('HTTP code "{0}" unhandled'.format(resp.code)) logging.debug('cinp: got HTTP code "{0}"'.format(http_code)) if http_code == 401: resp.close() logging.warning('cinp: Invalid Session') raise InvalidSession() if http_code == 403: resp.close() logging.warning('cinp: Not Authorized') raise NotAuthorized() if http_code == 404: resp.close() logging.warning('cinp: Not Found') raise NotFound() buff = str(resp.read(), 'utf-8').strip() if not buff: data = None else: try: data = json.loads(buff) except ValueError: data = None if http_code not in ( 400, 500): # these two codes can deal with non dict data logging.warning( 'cinp: Unable to parse response "{0}"'.format( buff[0:200])) raise ResponseError( 'Unable to parse response "{0}"'.format(buff[0:200])) header_map = {} for item in ('Position', 'Count', 'Total', 'Type', 'Multi-Object', 'Object-Id', 'verb'): try: header_map[item] = resp.headers[item] except KeyError: pass resp.close() if http_code == 400: try: data['message'] e = DetailedInvalidRequest(data) except (KeyError, ValueError, TypeError): e = InvalidRequest(buff[0:200]) logging.warning('cinp: Invalid Request "{0}"'.format(e.args[0])) raise e if http_code == 500: if isinstance(data, dict): try: message = 'Server Error "{0}"\n{1}'.format( data['message'], data['trace']) except KeyError: try: message = 'Server Error "{0}"'.format(data['message']) except KeyError: message = data else: message = 'Server Error: "{0}"'.format(buff[0:200]) logging.error('cinp: {0}'.format(message)) raise ServerError(message) return (http_code, data, header_map) def setAuth(self, auth_id=None, auth_token=None): """ Sets the Authencation id and token headers, call with out paramaters to remove the headers. """ if not auth_id: logging.debug('cinp: removing auth info') for index in range(len(self.opener.addheaders) - 1, -1, -1): if self.opener.addheaders[index][0] in ('Auth-Id', 'Auth-Token'): del self.opener.addheaders[index] else: logging.debug('cinp: setting auth info, id "{0}"'.format(auth_id)) self.opener.addheaders += [('Auth-Id', auth_id), ('Auth-Token', auth_token)] def describe(self, uri, timeout=30, retry_count=0, retry_on=None): """ DESCRIE """ logging.debug('cinp: DESCRIBE "{0}"'.format(uri)) (http_code, data, header_map) = self._request('DESCRIBE', uri, timeout=timeout, retry_count=retry_count, retry_on=retry_on) if http_code != 200: logging.warning( 'cinp: Unexpected HTTP Code "{0}" for DESCRIBE'.format( http_code)) raise ResponseError( 'Unexpected HTTP Code "{0}" for DESCRIBE'.format(http_code)) return data, header_map['Type'] def list(self, uri, filter_name=None, filter_value_map=None, position=0, count=10, timeout=30, retry_count=0, retry_on=None): """ LIST """ if filter_value_map is None: filter_value_map = {} if not isinstance(filter_value_map, dict): raise InvalidRequest('list filter_value_map must be a dict') if not isinstance(position, int) or not isinstance( count, int) or position < 0 or count < 0: raise InvalidRequest( 'position and count must bot be int and greater than 0') header_map = { 'Position': str(position), 'Count': str(count) } # TODO set position and coint to default to None, and don't send them if npt specified, rely on the server's defaults if filter_name is not None: header_map['Filter'] = filter_name logging.debug('cinp: LIST "{0}" with filter "{1}"'.format( uri, filter_name)) (http_code, id_list, header_map) = self._request('LIST', uri, data=filter_value_map, header_map=header_map, timeout=timeout, retry_count=retry_count, retry_on=retry_on) if http_code != 200: logging.warning( 'cinp: Unexpected HTTP Code "{0}" for LIST'.format(http_code)) raise ResponseError( 'Unexpected HTTP Code "{0}" for LIST'.format(http_code)) if not isinstance(id_list, list): logging.warning('cinp: Response id_list must be a list for LIST') raise ResponseError('Response id_list must be a list for LIST') count_map = {'position': 0, 'count': 0, 'total': 0} for item in ( 'Position', 'Count', 'Total' ): # NOTE HTTP Headers typicaly are CamelCase, but we want lower case for our count dictomary try: count_map[item.lower()] = int(header_map[item]) except (KeyError, ValueError): pass return (id_list, count_map) def get(self, uri, force_multi_mode=False, timeout=30, retry_count=0, retry_on=None): """ GET """ header_map = {} if force_multi_mode: header_map['Multi-Object'] = True logging.debug('cinp: GET "{0}"'.format(uri)) (http_code, rec_values, header_map) = self._request('GET', uri, header_map=header_map, timeout=timeout, retry_count=retry_count, retry_on=retry_on) if http_code != 200: logging.warning( 'cinp: Unexpected HTTP Code "{0}" for GET'.format(http_code)) raise ResponseError( 'Unexpected HTTP Code "{0}" for GET'.format(http_code)) if not isinstance(rec_values, dict): logging.warning('cinp: Response rec_values must be a dict for GET') raise ResponseError('Response rec_values must be a dict for GET') return rec_values def create(self, uri, values, timeout=30, retry_count=0, retry_on=None): """ CREATE """ if not isinstance(values, dict): raise InvalidRequest('values must be a dict') logging.debug('cinp: CREATE "{0}"'.format(uri)) (http_code, rec_values, header_map) = self._request('CREATE', uri, data=values, timeout=timeout, retry_count=retry_count, retry_on=retry_on) if http_code != 201: logging.warning( 'cinp: Unexpected HTTP Code "{0}" for CREATE'.format( http_code)) raise ResponseError( 'Unexpected HTTP Code "{0}" for CREATE'.format(http_code)) if not isinstance(rec_values, dict): logging.warning( 'cinp: Response rec_values must be a dict for CREATE') raise ResponseError( 'Response rec_values must be a dict for CREATE') try: object_id = header_map['Object-Id'] except KeyError: raise ResponseError('Object-Id header missing') return (object_id, rec_values) def update(self, uri, values, force_multi_mode=False, timeout=30, retry_count=0, retry_on=None): """ UPDATE """ if not isinstance(values, dict): raise InvalidRequest('values must be a dict') header_map = {} if force_multi_mode: header_map['Multi-Object'] = True logging.debug('cinp: UPDATE "{0}"'.format(uri)) (http_code, rec_values, _) = self._request('UPDATE', uri, data=values, header_map=header_map, timeout=timeout, retry_count=retry_count, retry_on=retry_on) if http_code != 200: logging.warning( 'cinp: Unexpected HTTP Code "{0}" for UPDATE'.format( http_code)) raise ResponseError( 'Unexpected HTTP Code "{0}" for UPDATE'.format(http_code)) if not isinstance(rec_values, dict): logging.warning( 'cinp: Response rec_values must be a dict for UPDATE') raise ResponseError('Response rec_values must be a str for UPDATE') return rec_values def delete(self, uri, timeout=30, retry_count=0, retry_on=None): """ DELETE """ logging.debug('cinp: DELETE "{0}"'.format(uri)) (http_code, _, _) = self._request('DELETE', uri, timeout=timeout, retry_count=retry_count, retry_on=retry_on) if http_code != 200: logging.warning( 'cinp: Unexpected HTTP Code "{0}" for DELETE'.format( http_code)) raise ResponseError( 'Unexpected HTTP Code "{0}" for DELETE'.format(http_code)) return True def call(self, uri, args, force_multi_mode=False, timeout=30, retry_count=0, retry_on=None): """ CALL """ if not isinstance(args, dict): raise InvalidRequest('args must be a dict') header_map = {} if force_multi_mode: header_map['Multi-Object'] = True logging.debug('cinp: CALL "{0}"'.format(uri)) (http_code, return_value, _) = self._request('CALL', uri, data=args, header_map=header_map, timeout=timeout, retry_count=retry_count, retry_on=retry_on) if http_code != 200: logging.warning( 'cinp: Unexpected HTTP Code "{0}" for CALL'.format(http_code)) raise ResponseError( 'Unexpected HTTP Code "{0}" for CALL'.format(http_code)) return return_value def getMulti(self, uri, id_list=None, chunk_size=10, retry_count=0, retry_on=None): """ returns a generator that will iterate over the uri/id_list, reterieving from the server in chunk_size blocks each item is ( rec_id, rec_values ) if uri is a list, id_list is ignored """ if isinstance(uri, list): id_list = [] for item in uri: (_, _, _, tmp_id_list, _) = self.uri.split(item) id_list += tmp_id_list (namespace, model, _, tmp_id_list, _) = self.uri.split(uri[0]) else: (namespace, model, _, tmp_id_list, _) = self.uri.split(uri) if id_list is None: id_list = tmp_id_list result_list = [] pos = 0 while pos < len(id_list): tmp_data = self.get(self.uri.build(namespace, model, None, id_list[pos:pos + chunk_size]), force_multi_mode=True, retry_count=retry_count, retry_on=retry_on) pos += chunk_size for key in tmp_data: result_list.append((key, tmp_data[key])) while len(result_list) > 0: yield result_list.pop(0) def getFilteredObjects(self, uri, filter_name=None, filter_value_map=None, list_chunk_size=100, get_chunk_size=10, timeout=30, retry_count=0, retry_on=None): pos = 0 total = 1 while pos < total: (tmp_id_list, count_map) = self.list(uri, filter_name=filter_name, filter_value_map=filter_value_map, position=pos, count=list_chunk_size, retry_count=retry_count, retry_on=retry_on) id_list = self.uri.extractIds(tmp_id_list) pos = count_map['position'] + count_map['count'] total = count_map['total'] return self.getMulti( uri, id_list, get_chunk_size, retry_count=retry_count, retry_on=retry_on ) # TODO: need to found out how to get the next chunk, return/yeild something def getFilteredURIs(self, uri, filter_name=None, filter_value_map=None, list_chunk_size=100, get_chunk_size=10, timeout=30, retry_count=0, retry_on=None): pos = 0 total = 1 while pos < total: (tmp_id_list, count_map) = self.list(uri, filter_name=filter_name, filter_value_map=filter_value_map, position=pos, count=list_chunk_size, retry_count=retry_count, retry_on=retry_on) id_list = self.uri.extractIds(tmp_id_list) pos = count_map['position'] + count_map['count'] total = count_map['total'] while len(id_list) > 0: yield id_list.pop(0) def getFile(self, uri, target_dir='/tmp', file_object=None, cb=None, timeout=30, chunk_size=(4096 * 1024)): """ if file_object is defined: The file contense are written to it and the filename as specified by the server is returned, None is returned if not filename is detected. The file_object is not closed. file_object must be opened with the 'b' attribute. Otherwise a file is created in target_dir, and the full path is returned. If the filename is not specified by the server, and a random filename is chosen. WARNING: there isn't checking done to make sure the target file does not allready exist, there is a possibility it could clober something that allready exists. we do make sure the filename fits a regex pattern that prevents it from escaping the target_dir. The "filename" as sent by the server is the "model" of the uri. make sure target_dir exists before calling getFile """ uri_parser = URI('/') try: # TODO: There has to be a better way to validate this uri (_, filename, _, _, _) = uri_parser.split(uri) except ValueError as e: raise InvalidRequest(str(e)) # Due to the return value we have to do our own request, this is pretty much a stright GET url = '{0}{1}'.format(self.host, uri) req = request.Request(url) req.get_method = lambda: 'GET' try: resp = self.opener.open(req, timeout=timeout) except request.HTTPError as e: raise ResponseError('HTTPError "{0}"'.format(e)) except request.URLError as e: if isinstance(e.reason, socket.timeout): raise Timeout( 'Request Timeout after {0} seconds'.format(timeout)) raise ResponseError('URLError "{0}" for "{1}" via "{2}"'.format( e, url, self.proxy)) http_code = resp.code if http_code != 200: logging.warning( 'cinp: Unexpected HTTP Code "{0}" for File Get'.format( http_code)) raise ResponseError( 'Unexpected HTTP Code "{0}" for File Get'.format(http_code)) try: size = resp.headers['Content-Length'] except KeyError: size = 0 if file_object is not None: file_writer = file_object else: if filename is None: file_writer = NamedTemporaryFile(dir=target_dir, mode='wb') filename = file_writer.name else: filename = os.path.join(target_dir, filename) file_writer = open(filename, 'wb') buff = resp.read(chunk_size) while buff: file_writer.write(buff) if cb: cb(file_writer.tell(), size) buff = resp.read(chunk_size) resp.close() if file_object is not None: return filename else: file_writer.close() return filename def uploadFile(self, uri, filepath, filename=None, cb=None, timeout=30): """ filepath can be a string of the path name or a file object. If a file object either specify the filename or make sure your file object exposes the attribute 'name'. Also if file object, must be opened in binary mode, ie: 'rb' NOTE: this is not a CInP function, but a conviance function for uploading large files. """ uri_parser = URI('/') try: # TODO: There has to be a better way to validate this uri (namespace, model, action, id_list, _) = uri_parser.split(uri) except ValueError as e: raise InvalidRequest(str(e)) if action is not None or id_list is not None: raise InvalidRequest( 'file upload target can\'t be an action nor have ids') if isinstance(filepath, str): if filename is None: filename = os.path.basename(filepath) file_reader = _readerWrapper(open(filepath, 'rb'), cb) else: if filename is None: filename = os.path.basename(filepath.name) file_reader = _readerWrapper(filepath, cb) header_map = { 'Content-Disposition': 'inline: filename="{0}"'.format(filename), 'Content-Length': len(file_reader) } (http_code, data, _) = self._request('UPLOAD', uri_parser.build(namespace, model), data=file_reader, header_map=header_map, timeout=timeout) if http_code != 202: logging.warning( 'cinp: Unexpected HTTP Code "{0}" for File Upload'.format( http_code)) raise ResponseError( 'Unexpected HTTP Code "{0}" for File Upload'.format(http_code)) return data['uri']
def getFile(self, uri, target_dir='/tmp', file_object=None, cb=None, timeout=30, chunk_size=(4096 * 1024)): """ if file_object is defined: The file contense are written to it and the filename as specified by the server is returned, None is returned if not filename is detected. The file_object is not closed. file_object must be opened with the 'b' attribute. Otherwise a file is created in target_dir, and the full path is returned. If the filename is not specified by the server, and a random filename is chosen. WARNING: there isn't checking done to make sure the target file does not allready exist, there is a possibility it could clober something that allready exists. we do make sure the filename fits a regex pattern that prevents it from escaping the target_dir. The "filename" as sent by the server is the "model" of the uri. make sure target_dir exists before calling getFile """ uri_parser = URI('/') try: # TODO: There has to be a better way to validate this uri (_, filename, _, _, _) = uri_parser.split(uri) except ValueError as e: raise InvalidRequest(str(e)) # Due to the return value we have to do our own request, this is pretty much a stright GET url = '{0}{1}'.format(self.host, uri) req = request.Request(url) req.get_method = lambda: 'GET' try: resp = self.opener.open(req, timeout=timeout) except request.HTTPError as e: raise ResponseError('HTTPError "{0}"'.format(e)) except request.URLError as e: if isinstance(e.reason, socket.timeout): raise Timeout( 'Request Timeout after {0} seconds'.format(timeout)) raise ResponseError('URLError "{0}" for "{1}" via "{2}"'.format( e, url, self.proxy)) http_code = resp.code if http_code != 200: logging.warning( 'cinp: Unexpected HTTP Code "{0}" for File Get'.format( http_code)) raise ResponseError( 'Unexpected HTTP Code "{0}" for File Get'.format(http_code)) try: size = resp.headers['Content-Length'] except KeyError: size = 0 if file_object is not None: file_writer = file_object else: if filename is None: file_writer = NamedTemporaryFile(dir=target_dir, mode='wb') filename = file_writer.name else: filename = os.path.join(target_dir, filename) file_writer = open(filename, 'wb') buff = resp.read(chunk_size) while buff: file_writer.write(buff) if cb: cb(file_writer.tell(), size) buff = resp.read(chunk_size) resp.close() if file_object is not None: return filename else: file_writer.close() return filename
def test_urilist_to_uri(): uri = URI('/api/v1/') id_list = [ '/api/v1/ns/model:sdf:', '/api/v1/ns/model:234:', '/api/v1/ns/model:rfv:' ] assert uri.uriListToMultiURI(id_list) == '/api/v1/ns/model:sdf:234:rfv:' id_list = [ '/api/v1/ns/model:sdf:', '/api/v1/ns/model:234:www:', '/api/v1/ns/model:rfv:' ] assert uri.uriListToMultiURI( id_list) == '/api/v1/ns/model:sdf:234:www:rfv:' id_list = ['/api/v1/ns/model:234:www:'] assert uri.uriListToMultiURI(id_list) == '/api/v1/ns/model:234:www:' id_list = [ '/api/v1/ns/model:sdf:', '/api/v1/ns/model:234:www', '/api/v1/ns/model:rfv:' ] with pytest.raises(ValueError): uri.uriListToMultiURI(id_list) id_list = ['/api/v1/ns/model'] uri.uriListToMultiURI(id_list) == [] id_list = [ '/api/v1/ns/model:sdf:', '/api/v1/ns/model', '/api/v1/ns/model:rfv:' ] uri.uriListToMultiURI(id_list) == '/api/v1/ns/model:sdf:rfv:' assert uri.uriListToMultiURI([]) == []