def __missing__(self, key): """Overrides the default KeyError exception.""" if len(self.keys()) == 0: raise BrainError('No URL Map found. Cannot make remote call') else: raise BrainError('Key {0} not found in urlmap.'.format(key))
def _sendRequest(self, request_type): ''' sends the api requests to the server ''' assert request_type in ['get', 'post'], 'Valid request types are "get" and "post".' # Loads the local config parameters self._loadConfigParams() # Send the request try: if request_type == 'get': self._response = self.session.get(self.url, params=self.params, timeout=self.timeout, headers=self.headers, stream=self.stream, verify=self.verify) elif request_type == 'post': self._response = self.session.post(self.url, data=self.params, timeout=self.timeout, headers=self.headers, stream=self.stream, verify=self.verify) except requests.Timeout as rt: self._closeRequestSession() errmsg = 'Your request took longer than 5 minutes and timed out. Please try again or simplify your request.' raise BrainError('Requests Timeout Error: {0}\n{1}'.format(rt, errmsg)) except requests.URLRequired as urlreq: self._closeRequestSession() raise BrainError('Requests Valid URL Required: {0}'.format(urlreq)) except requests.ConnectionError as con: self._closeRequestSession() raise BrainError('Requests Connection Error: {0}'.format(con)) except requests.RequestException as req: self._closeRequestSession() raise BrainError('Ambiguous Requests Error: {0}'.format(req)) else: # Check the response if it's good self._checkResponse(self._response)
def __init__(self, route, params=None, request_type='post', auth='token', timeout=(3.05, 300), headers=None, stream=None, datastream=None, send=True, base=None, verify=True): self.results = None self.response_time = None self.route = route self.params = params self.request_type = request_type self.timeout = timeout self.stream = stream self.verify = verify self.datastream = datastream self.headers = headers if headers is not None else {} self.statuscodes = {200: 'Ok', 401: 'Authentication Required', 404: 'URL Not Found', 500: 'Internal Server Error', 405: 'Method Not Allowed', 400: 'Bad Request', 502: 'Bad Gateway', 504: 'Gateway Timeout', 422: 'Unprocessable Entity', 429: 'Rate Limit Exceeded'} self.compression = self.params['compression'] if self.params and \ 'compression' in self.params else bconfig.compression base = base if base else bconfig.sasurl self.url = urljoin(base, route) if self.route else None # set request Session self._setRequestSession() # set authentication self.setAuth(authtype=auth) # sends the request if self.url and send: self._sendRequest(request_type) elif not self.url and send: raise BrainError('No route and/or url specified {0}'.format(self.url))
def _compress_msgpack(data, uncompress=None): ''' Compress/Uncompress msgpack data ''' # import the package try: import msgpack import msgpack_numpy as m except ImportError as e: compress_with = 'json' raise BrainWarning( 'Must have Python packages msgpack and msgpack_numpy ' 'installed to use msgpack compression. Defaulting to json') else: m.patch() # do the compression try: if uncompress: comp_data = msgpack.unpackb(data, raw=False) else: comp_data = msgpack.packb(data, use_bin_type=True) except Exception as e: raise BrainError('Cannot (un)compress msgpack data. {0}'.format(e)) else: return comp_data
def wrapper(*args, **kwargs): # iscollab = bconfig.access == 'collab' valid_netrc = bconfig._check_netrc() if valid_netrc: return func(*args, **kwargs) else: raise BrainError( 'You are not authorized to access the SDSS collaboration')
def _set_defaults(self, logLevel='WARNING', logFileLevel='INFO', logFilePath='~/.brain/brain.log', mode='append', wrapperLength=70): """Reset logger to its initial state.""" # Remove all previous handlers for handler in self.handlers[:]: self.removeHandler(handler) # Set levels self.setLevel('DEBUG') # Set up the stdout handler self.sh = logging.StreamHandler() self.sh.emit = self._stream_formatter self.addHandler(self.sh) self.wrapperLength = wrapperLength # Set up the main log file handler if requested (but this might fail if # configuration directory or log file is not writeable). logFilePath = os.path.expanduser(logFilePath) logDir = os.path.dirname(logFilePath) if not os.path.exists(logDir): os.mkdir(logDir) try: if mode.lower() == 'overwrite': self.fh = FileHandler(logFilePath, mode='w') elif mode.lower() == 'append': self.fh = TimedRotatingFileHandler( logFilePath, when='midnight', utc=True) else: raise BrainError('logger mode {0} not recognised'.format(mode)) except (IOError, OSError) as e: warnings.warn( 'log file {0!r} could not be opened for writing: ' '{1}'.format(logFilePath, unicode(e)), RuntimeWarning) else: self.fh.setFormatter(fmt) self.addHandler(self.fh) # Adds a header only to the file handler self.sh.setLevel(logging.CRITICAL) self.fh.setLevel(logging.DEBUG) self.debug('') self.debug('--------------------------------') self.debug('----- Restarting logger. -------') self.debug('--------------------------------') self.sh.setLevel(logLevel) self.fh.setLevel(logFileLevel) self.logFilename = logFilePath
def _read_netrc(self, host): ''' Read the netrc file for a given host ''' if not self._check_netrc(): raise BrainError('netrc did not pass checks. Cannot read!') assert host in self._valid_hosts and self._valid_hosts[host], '{0} must be a valid host in the netrc'.format(host) netfile = netrc.netrc(self._netrc_path) user, acct, passwd = netfile.authenticators(host) return user, passwd
def _check_netrc(self): """Makes sure there is a valid netrc.""" if not os.path.exists(self._netrc_path): raise BrainError('No .netrc file found in your HOME directory!') else: if oct(os.stat(self._netrc_path).st_mode)[-3:] != '600': raise BrainError('your .netrc file does not have 600 permissions. Please fix it by ' 'running chmod 600 ~/.netrc. Authentication will not work with ' 'permissions different from 600.') # read the netrc file netfile = netrc.netrc(self._netrc_path) # check the hosts self._check_host('data.sdss.org', netfile, msg='You will not be able to download SDSS data') self._check_host('api.sdss.org', netfile, msg='You will not have remote access to SDSS data') # validate if any are good return any(self._valid_hosts.values())
def _compress_json(data, uncompress=None): ''' Compress/Uncompress JSON data ''' try: if uncompress: comp_data = json.loads(data) else: comp_data = json.dumps(data) except Exception as e: raise BrainError('Cannot (un)compress JSON data. {0}'.format(e)) else: return comp_data
def _checkAuth(self): ''' Checks the API for authentication Main user and token authentication take care of in each request. See decorators applied to marvin.api.base ''' # don't check authentication for these public decorated routes ispublic = getattr(current_app.view_functions[request.endpoint], 'is_public', False) if ispublic: return if 'Authorization' not in request.headers: raise BrainError('Authorization is required to access!')
def build_routemap(app): ''' Builds a Flask Web App's dictionary of routes Constructs a dictionary containing all the routes defined inside a given Flask Web App. The route endpoints are deconstructed into a set of nested dictionaries of the form [blueprint][endpoint], which contains a methods and a url key. The url key returns the full route path. E.g. the API route to get a cube, which has a name "getCube" is expressed as ['api']['getCube']. To access the url, ['api']['getCube']['url'] returns "/marvin/api/cubes/{name}/" Parameters: app (Flask Application): The Flask app to extract routes from Returns: A dict of all routes ''' output = {} for rule in app.url_map.iter_rules(): # get options options = {} for arg in rule.arguments: options[arg] = '[{0}]'.format(arg) options['_external'] = False # get endpoint fullendpoint = rule.endpoint esplit = fullendpoint.split('.') grp, endpoint = esplit[0], None if len(esplit) == 1 else esplit[1] output.setdefault(grp, {}).update({endpoint: {}}) # get methods methods = ','.join(rule.methods) output[grp][endpoint]['methods'] = methods # build url try: rawurl = url_for(fullendpoint, **options) except ValueError as e: raise BrainError('Error generating url for {0}: {1}'.format( fullendpoint, e)) url = unquote(rawurl).replace('[', '{').replace(']', '}') output[grp][endpoint]['url'] = url return output
def compress_data(data, compress_with=None, uncompress=None): ''' Compress data via json or msgpack Parameters: data (obj) The input data to compress or uncompress compress_with (str): Compression algorithm. Defaults to config.compression. uncompress (bool): If True, uncompresses the data instead of compressing. Default is False Returns: Data compressed with with json or msgpack ''' from brain import bconfig # check compression if not compress_with: compress_with = bconfig.compression if compress_with == 'msgpack': try: import msgpack except ImportError as e: compress_with = 'json' raise BrainWarning( 'Must have Python packages msgpack and msgpack_numpy ' 'installed to use msgpack compression. Defaulting to json' ) assert compress_with in bconfig._compression_types, 'compress_with must be one of {0}'.format( bconfig._compression_types) # compress the data if compress_with == 'json': comp_data = _compress_json(data, uncompress=uncompress) elif compress_with == 'msgpack': comp_data = _compress_msgpack(data, uncompress=uncompress) else: raise BrainError( 'Unrecognized compression algorithm {0}'.format(compress_with)) return comp_data
def _checkResponse(self, response): ''' Checks the response for proper http status code ''' # check for bad status try: isbad = response.raise_for_status() except requests.HTTPError as http: self.status_code = response.status_code errmsg = 'Error accessing {0}: {1}-{2}'.format( response.url, response.status_code, self.statuscodes[response.status_code]) self.results = { 'http_status_code': response.status_code, 'message': errmsg } json_data = self._get_content(response) if self.status_code == 401: if self.authtype == 'netrc': msg = 'Please create or check credentials in your local .netrc file' elif self.authtype == 'oauth': msg = 'Please check your Oauth authentication' elif self.authtype == 'http': msg = 'Please check your http/digest authentication' elif self.authtype == 'token': msg = '{0}. Please check your token or login again for a fresh one.'.format( json_data['msg']) else: msg = 'Please check your authentication method.' errmsg = json_data['error'] if 'error' in json_data else '' self._closeRequestSession() raise BrainApiAuthError( 'API Authentication Error: {0}. {1}'.format(msg, errmsg)) elif self.status_code == 422: self._closeRequestSession() raise BrainError( 'Requests Http Status Error: {0}\nValidation Errors:\n{1}'. format(http, json_data)) else: self._closeRequestSession() if 'api_error' in json_data: apijson = json_data['api_error'] errmsg = '{0}\n{1}'.format( apijson['message'], apijson['traceback'] ) if 'message' in apijson else '{0}'.format( apijson['traceback']) elif 'error' in json_data: err = json_data['error'] errmsg = '{0}'.format(err) raise BrainError('Requests Http Status Error: {0}\n{1}'.format( http, errmsg)) else: # Not bad assert isbad is None, 'Http status code should not be bad' assert response.ok is True, 'Ok status should be true' # if 'route' not in self.url and 'clean' not in self.url: # self._closeRequestSession() # raise BrainError('test error is now raised here') self.status_code = response.status_code self.results = self._get_content(response) self.response_time = response.elapsed if not self.results: self.results = response.text self._closeRequestSession() raise BrainError('Response not in JSON format. {0}'.format( self.results)) # Raises an error if status is -1 if 'status' in self.results and self.results['status'] == -1: errorMsg = 'no error message provided' \ if 'error' not in self.results else self.results['error'] self._check_for_traceback() self._closeRequestSession() raise BrainError( 'Something went wrong on the server side: {0}'.format( errorMsg))
def wrapper(*args, **kwargs): if not Path: raise BrainError('sdss_access is not installed') else: return func(*args, **kwargs)
def buildRouteMap(self): """ Build the URL route map for all routes in the Flask app. .. :quickref: General; Returns the urlmap dictionary of Marvin API routes Syntax of output: {"api": {blueprint: {endpoint: {'methods':x, 'url':x} } } :form release: the release of MaNGA :resjson int status: status of response. 1 if good, -1 if bad. :resjson string error: error message, null if None :resjson json inconfig: json of incoming configuration :resjson json utahconfig: json of outcoming configuration :resjson string traceback: traceback of an error, null if None :resjson json data: dictionary of returned data :json dict urlmap: dict of the Marvin API routes :resheader Content-Type: application/json :statuscode 200: no error :statuscode 422: invalid input parameters :raises BrainError: Raised when url_for can't format the endpoint name into a valid url. **Example request**: .. sourcecode:: http GET /marvin/api/general/getroutemap/ HTTP/1.1 Host: api.sdss.org Accept: application/json, */* **Example response**: .. sourcecode:: http HTTP/1.1 200 OK Content-Type: application/json { "status": 1, "error": null, "inconfig": {"release": "MPL-5"}, "utahconfig": {"release": "MPL-5", "mode": "local"}, "traceback": null, "data": {"urlmap": {"api": {"CubeView:index": {"methods": "HEAD,OPTIONS,GET","url": "/marvin2/api/cubes/"}, ... } } } """ output = {} for rule in current_app.url_map.iter_rules(): # get options options = {} for arg in rule.arguments: options[arg] = '[{0}]'.format(arg) options['_external'] = False # get endpoint fullendpoint = rule.endpoint esplit = fullendpoint.split('.') grp, endpoint = esplit[0], None if len(esplit) == 1 else esplit[1] output.setdefault(grp, {}).update({endpoint: {}}) # get methods methods = ','.join(rule.methods) output[grp][endpoint]['methods'] = methods # build url try: rawurl = url_for(fullendpoint, **options) except ValueError as e: raise BrainError('Error generating url for {0}: {1}'.format( fullendpoint, e)) url = unquote(rawurl).replace('[', '{').replace(']', '}') output[grp][endpoint]['url'] = url res = {'urlmap': output, 'status': 1} self.update_results(res) return jsonify(self.results)