def __missing__(self, key):
"""Overrides the default KeyError exception."""
if len(self.keys()) == 0:
raise BrainError('No URL Map found. Cannot make remote call')
else:
raise BrainError('Key {0} not found in urlmap.'.format(key))
def _sendRequest(self, request_type):
''' sends the api requests to the server '''
assert request_type in ['get', 'post'], 'Valid request types are "get" and "post".'
# Loads the local config parameters
self._loadConfigParams()
# Send the request
try:
if request_type == 'get':
self._response = self.session.get(self.url, params=self.params, timeout=self.timeout,
headers=self.headers, stream=self.stream, verify=self.verify)
elif request_type == 'post':
self._response = self.session.post(self.url, data=self.params, timeout=self.timeout,
headers=self.headers, stream=self.stream, verify=self.verify)
except requests.Timeout as rt:
self._closeRequestSession()
errmsg = 'Your request took longer than 5 minutes and timed out. Please try again or simplify your request.'
raise BrainError('Requests Timeout Error: {0}\n{1}'.format(rt, errmsg))
except requests.URLRequired as urlreq:
self._closeRequestSession()
raise BrainError('Requests Valid URL Required: {0}'.format(urlreq))
except requests.ConnectionError as con:
self._closeRequestSession()
raise BrainError('Requests Connection Error: {0}'.format(con))
except requests.RequestException as req:
self._closeRequestSession()
raise BrainError('Ambiguous Requests Error: {0}'.format(req))
else:
# Check the response if it's good
self._checkResponse(self._response)
def __init__(self, route, params=None, request_type='post', auth='token',
timeout=(3.05, 300), headers=None, stream=None,
datastream=None, send=True, base=None, verify=True):
self.results = None
self.response_time = None
self.route = route
self.params = params
self.request_type = request_type
self.timeout = timeout
self.stream = stream
self.verify = verify
self.datastream = datastream
self.headers = headers if headers is not None else {}
self.statuscodes = {200: 'Ok', 401: 'Authentication Required', 404: 'URL Not Found',
500: 'Internal Server Error', 405: 'Method Not Allowed',
400: 'Bad Request', 502: 'Bad Gateway', 504: 'Gateway Timeout',
422: 'Unprocessable Entity', 429: 'Rate Limit Exceeded'}
self.compression = self.params['compression'] if self.params and \
'compression' in self.params else bconfig.compression
base = base if base else bconfig.sasurl
self.url = urljoin(base, route) if self.route else None
# set request Session
self._setRequestSession()
# set authentication
self.setAuth(authtype=auth)
# sends the request
if self.url and send:
self._sendRequest(request_type)
elif not self.url and send:
raise BrainError('No route and/or url specified {0}'.format(self.url))
def _compress_msgpack(data, uncompress=None):
''' Compress/Uncompress msgpack data '''
# import the package
try:
import msgpack
import msgpack_numpy as m
except ImportError as e:
compress_with = 'json'
raise BrainWarning(
'Must have Python packages msgpack and msgpack_numpy '
'installed to use msgpack compression. Defaulting to json')
else:
m.patch()
# do the compression
try:
if uncompress:
comp_data = msgpack.unpackb(data, raw=False)
else:
comp_data = msgpack.packb(data, use_bin_type=True)
except Exception as e:
raise BrainError('Cannot (un)compress msgpack data. {0}'.format(e))
else:
return comp_data
def wrapper(*args, **kwargs):
# iscollab = bconfig.access == 'collab'
valid_netrc = bconfig._check_netrc()
if valid_netrc:
return func(*args, **kwargs)
else:
raise BrainError(
'You are not authorized to access the SDSS collaboration')
def _set_defaults(self,
logLevel='WARNING',
logFileLevel='INFO',
logFilePath='~/.brain/brain.log',
mode='append',
wrapperLength=70):
"""Reset logger to its initial state."""
# Remove all previous handlers
for handler in self.handlers[:]:
self.removeHandler(handler)
# Set levels
self.setLevel('DEBUG')
# Set up the stdout handler
self.sh = logging.StreamHandler()
self.sh.emit = self._stream_formatter
self.addHandler(self.sh)
self.wrapperLength = wrapperLength
# Set up the main log file handler if requested (but this might fail if
# configuration directory or log file is not writeable).
logFilePath = os.path.expanduser(logFilePath)
logDir = os.path.dirname(logFilePath)
if not os.path.exists(logDir):
os.mkdir(logDir)
try:
if mode.lower() == 'overwrite':
self.fh = FileHandler(logFilePath, mode='w')
elif mode.lower() == 'append':
self.fh = TimedRotatingFileHandler(
logFilePath, when='midnight', utc=True)
else:
raise BrainError('logger mode {0} not recognised'.format(mode))
except (IOError, OSError) as e:
warnings.warn(
'log file {0!r} could not be opened for writing: '
'{1}'.format(logFilePath, unicode(e)), RuntimeWarning)
else:
self.fh.setFormatter(fmt)
self.addHandler(self.fh)
# Adds a header only to the file handler
self.sh.setLevel(logging.CRITICAL)
self.fh.setLevel(logging.DEBUG)
self.debug('')
self.debug('--------------------------------')
self.debug('----- Restarting logger. -------')
self.debug('--------------------------------')
self.sh.setLevel(logLevel)
self.fh.setLevel(logFileLevel)
self.logFilename = logFilePath
def _read_netrc(self, host):
''' Read the netrc file for a given host '''
if not self._check_netrc():
raise BrainError('netrc did not pass checks. Cannot read!')
assert host in self._valid_hosts and self._valid_hosts[host], '{0} must be a valid host in the netrc'.format(host)
netfile = netrc.netrc(self._netrc_path)
user, acct, passwd = netfile.authenticators(host)
return user, passwd
def _check_netrc(self):
"""Makes sure there is a valid netrc."""
if not os.path.exists(self._netrc_path):
raise BrainError('No .netrc file found in your HOME directory!')
else:
if oct(os.stat(self._netrc_path).st_mode)[-3:] != '600':
raise BrainError('your .netrc file does not have 600 permissions. Please fix it by '
'running chmod 600 ~/.netrc. Authentication will not work with '
'permissions different from 600.')
# read the netrc file
netfile = netrc.netrc(self._netrc_path)
# check the hosts
self._check_host('data.sdss.org', netfile, msg='You will not be able to download SDSS data')
self._check_host('api.sdss.org', netfile, msg='You will not have remote access to SDSS data')
# validate if any are good
return any(self._valid_hosts.values())
def _compress_json(data, uncompress=None):
''' Compress/Uncompress JSON data '''
try:
if uncompress:
comp_data = json.loads(data)
else:
comp_data = json.dumps(data)
except Exception as e:
raise BrainError('Cannot (un)compress JSON data. {0}'.format(e))
else:
return comp_data
def _checkAuth(self):
''' Checks the API for authentication
Main user and token authentication take care of in each request.
See decorators applied to marvin.api.base
'''
# don't check authentication for these public decorated routes
ispublic = getattr(current_app.view_functions[request.endpoint], 'is_public', False)
if ispublic:
return
if 'Authorization' not in request.headers:
raise BrainError('Authorization is required to access!')
def build_routemap(app):
''' Builds a Flask Web App's dictionary of routes
Constructs a dictionary containing all the routes defined
inside a given Flask Web App. The route endpoints are deconstructed into
a set of nested dictionaries of the form [blueprint][endpoint], which
contains a methods and a url key. The url key returns the full route path.
E.g. the API route to get a cube, which has a name "getCube" is expressed as
['api']['getCube']. To access the url, ['api']['getCube']['url'] returns
"/marvin/api/cubes/{name}/"
Parameters:
app (Flask Application):
The Flask app to extract routes from
Returns:
A dict of all routes
'''
output = {}
for rule in app.url_map.iter_rules():
# get options
options = {}
for arg in rule.arguments:
options[arg] = '[{0}]'.format(arg)
options['_external'] = False
# get endpoint
fullendpoint = rule.endpoint
esplit = fullendpoint.split('.')
grp, endpoint = esplit[0], None if len(esplit) == 1 else esplit[1]
output.setdefault(grp, {}).update({endpoint: {}})
# get methods
methods = ','.join(rule.methods)
output[grp][endpoint]['methods'] = methods
# build url
try:
rawurl = url_for(fullendpoint, **options)
except ValueError as e:
raise BrainError('Error generating url for {0}: {1}'.format(
fullendpoint, e))
url = unquote(rawurl).replace('[', '{').replace(']', '}')
output[grp][endpoint]['url'] = url
return output
def compress_data(data, compress_with=None, uncompress=None):
''' Compress data via json or msgpack
Parameters:
data (obj)
The input data to compress or uncompress
compress_with (str):
Compression algorithm. Defaults to config.compression.
uncompress (bool):
If True, uncompresses the data instead of compressing. Default is False
Returns:
Data compressed with with json or msgpack
'''
from brain import bconfig
# check compression
if not compress_with:
compress_with = bconfig.compression
if compress_with == 'msgpack':
try:
import msgpack
except ImportError as e:
compress_with = 'json'
raise BrainWarning(
'Must have Python packages msgpack and msgpack_numpy '
'installed to use msgpack compression. Defaulting to json'
)
assert compress_with in bconfig._compression_types, 'compress_with must be one of {0}'.format(
bconfig._compression_types)
# compress the data
if compress_with == 'json':
comp_data = _compress_json(data, uncompress=uncompress)
elif compress_with == 'msgpack':
comp_data = _compress_msgpack(data, uncompress=uncompress)
else:
raise BrainError(
'Unrecognized compression algorithm {0}'.format(compress_with))
return comp_data
def _checkResponse(self, response):
''' Checks the response for proper http status code '''
# check for bad status
try:
isbad = response.raise_for_status()
except requests.HTTPError as http:
self.status_code = response.status_code
errmsg = 'Error accessing {0}: {1}-{2}'.format(
response.url, response.status_code,
self.statuscodes[response.status_code])
self.results = {
'http_status_code': response.status_code,
'message': errmsg
}
json_data = self._get_content(response)
if self.status_code == 401:
if self.authtype == 'netrc':
msg = 'Please create or check credentials in your local .netrc file'
elif self.authtype == 'oauth':
msg = 'Please check your Oauth authentication'
elif self.authtype == 'http':
msg = 'Please check your http/digest authentication'
elif self.authtype == 'token':
msg = '{0}. Please check your token or login again for a fresh one.'.format(
json_data['msg'])
else:
msg = 'Please check your authentication method.'
errmsg = json_data['error'] if 'error' in json_data else ''
self._closeRequestSession()
raise BrainApiAuthError(
'API Authentication Error: {0}. {1}'.format(msg, errmsg))
elif self.status_code == 422:
self._closeRequestSession()
raise BrainError(
'Requests Http Status Error: {0}\nValidation Errors:\n{1}'.
format(http, json_data))
else:
self._closeRequestSession()
if 'api_error' in json_data:
apijson = json_data['api_error']
errmsg = '{0}\n{1}'.format(
apijson['message'], apijson['traceback']
) if 'message' in apijson else '{0}'.format(
apijson['traceback'])
elif 'error' in json_data:
err = json_data['error']
errmsg = '{0}'.format(err)
raise BrainError('Requests Http Status Error: {0}\n{1}'.format(
http, errmsg))
else:
# Not bad
assert isbad is None, 'Http status code should not be bad'
assert response.ok is True, 'Ok status should be true'
# if 'route' not in self.url and 'clean' not in self.url:
# self._closeRequestSession()
# raise BrainError('test error is now raised here')
self.status_code = response.status_code
self.results = self._get_content(response)
self.response_time = response.elapsed
if not self.results:
self.results = response.text
self._closeRequestSession()
raise BrainError('Response not in JSON format. {0}'.format(
self.results))
# Raises an error if status is -1
if 'status' in self.results and self.results['status'] == -1:
errorMsg = 'no error message provided' \
if 'error' not in self.results else self.results['error']
self._check_for_traceback()
self._closeRequestSession()
raise BrainError(
'Something went wrong on the server side: {0}'.format(
errorMsg))
def wrapper(*args, **kwargs):
if not Path:
raise BrainError('sdss_access is not installed')
else:
return func(*args, **kwargs)
def buildRouteMap(self):
""" Build the URL route map for all routes in the Flask app.
.. :quickref: General; Returns the urlmap dictionary of Marvin API routes
Syntax of output: {"api": {blueprint: {endpoint: {'methods':x, 'url':x} } }
:form release: the release of MaNGA
:resjson int status: status of response. 1 if good, -1 if bad.
:resjson string error: error message, null if None
:resjson json inconfig: json of incoming configuration
:resjson json utahconfig: json of outcoming configuration
:resjson string traceback: traceback of an error, null if None
:resjson json data: dictionary of returned data
:json dict urlmap: dict of the Marvin API routes
:resheader Content-Type: application/json
:statuscode 200: no error
:statuscode 422: invalid input parameters
:raises BrainError: Raised when url_for can't format the endpoint name into a valid url.
**Example request**:
.. sourcecode:: http
GET /marvin/api/general/getroutemap/ HTTP/1.1
Host: api.sdss.org
Accept: application/json, */*
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Type: application/json
{
"status": 1,
"error": null,
"inconfig": {"release": "MPL-5"},
"utahconfig": {"release": "MPL-5", "mode": "local"},
"traceback": null,
"data": {"urlmap": {"api": {"CubeView:index": {"methods": "HEAD,OPTIONS,GET","url": "/marvin2/api/cubes/"},
...
}
}
}
"""
output = {}
for rule in current_app.url_map.iter_rules():
# get options
options = {}
for arg in rule.arguments:
options[arg] = '[{0}]'.format(arg)
options['_external'] = False
# get endpoint
fullendpoint = rule.endpoint
esplit = fullendpoint.split('.')
grp, endpoint = esplit[0], None if len(esplit) == 1 else esplit[1]
output.setdefault(grp, {}).update({endpoint: {}})
# get methods
methods = ','.join(rule.methods)
output[grp][endpoint]['methods'] = methods
# build url
try:
rawurl = url_for(fullendpoint, **options)
except ValueError as e:
raise BrainError('Error generating url for {0}: {1}'.format(
fullendpoint, e))
url = unquote(rawurl).replace('[', '{').replace(']', '}')
output[grp][endpoint]['url'] = url
res = {'urlmap': output, 'status': 1}
self.update_results(res)
return jsonify(self.results)