def api_from_config(version=None, apiconfig=None, **kwargs):
"""Return an apiclient Resources object enabling access to an Arvados server
instance.
:version:
A string naming the version of the Arvados REST API to use (for
example, 'v1').
:apiconfig:
If provided, this should be a dict-like object (must support the get()
method) with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and
optionally ARVADOS_API_HOST_INSECURE. If not provided, use
arvados.config (which gets these parameters from the environment by
default.)
Other keyword arguments such as `cache` will be passed along `api()`
"""
# Load from user configuration or environment
if apiconfig is None:
apiconfig = config.settings()
for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
if x not in apiconfig:
raise ValueError("%s is not set. Aborting." % x)
host = apiconfig.get('ARVADOS_API_HOST')
token = apiconfig.get('ARVADOS_API_TOKEN')
insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig)
return api(version=version, host=host, token=token, insecure=insecure, **kwargs)
def subscribe(api, filters, on_event, poll_fallback=15, last_log_id=None):
"""
:api:
a client object retrieved from arvados.api(). The caller should not use this client object for anything else after calling subscribe().
:filters:
Initial subscription filters.
:on_event:
The callback when a message is received.
:poll_fallback:
If websockets are not available, fall back to polling every N seconds. If poll_fallback=False, this will return None if websockets are not available.
:last_log_id:
Log rows that are newer than the log id
"""
if not poll_fallback:
return _subscribe_websocket(api, filters, on_event, last_log_id)
try:
if not config.flag_is_true('ARVADOS_DISABLE_WEBSOCKETS'):
return _subscribe_websocket(api, filters, on_event, last_log_id)
else:
_logger.info("Using polling because ARVADOS_DISABLE_WEBSOCKETS is true")
except Exception as e:
_logger.warn("Falling back to polling after websocket error: %s" % e)
p = PollClient(api, filters, on_event, poll_fallback, last_log_id)
p.start()
return p
def api_from_config(version=None, apiconfig=None, **kwargs):
"""Return an apiclient Resources object enabling access to an Arvados server
instance.
:version:
A string naming the version of the Arvados REST API to use (for
example, 'v1').
:apiconfig:
If provided, this should be a dict-like object (must support the get()
method) with entries for ARVADOS_API_HOST, ARVADOS_API_TOKEN, and
optionally ARVADOS_API_HOST_INSECURE. If not provided, use
arvados.config (which gets these parameters from the environment by
default.)
Other keyword arguments such as `cache` will be passed along `api()`
"""
# Load from user configuration or environment
if apiconfig is None:
apiconfig = config.settings()
for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
if x not in apiconfig:
raise ValueError("%s is not set. Aborting." % x)
host = apiconfig.get('ARVADOS_API_HOST')
token = apiconfig.get('ARVADOS_API_TOKEN')
insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE', apiconfig)
return api(version=version,
host=host,
token=token,
insecure=insecure,
**kwargs)
def subscribe(api, filters, on_event, poll_fallback=15, last_log_id=None):
"""
:api:
a client object retrieved from arvados.api(). The caller should not use this client object for anything else after calling subscribe().
:filters:
Initial subscription filters.
:on_event:
The callback when a message is received.
:poll_fallback:
If websockets are not available, fall back to polling every N seconds. If poll_fallback=False, this will return None if websockets are not available.
:last_log_id:
Log rows that are newer than the log id
"""
if not poll_fallback:
return _subscribe_websocket(api, filters, on_event, last_log_id)
try:
if not config.flag_is_true('ARVADOS_DISABLE_WEBSOCKETS'):
return _subscribe_websocket(api, filters, on_event, last_log_id)
else:
_logger.info(
"Using polling because ARVADOS_DISABLE_WEBSOCKETS is true")
except Exception as e:
_logger.warn("Falling back to polling after websocket error: %s" % e)
p = PollClient(api, filters, on_event, poll_fallback, last_log_id)
p.start()
return p
def __init__(self, url, filters, on_event):
ssl_options = {'ca_certs': arvados.util.ca_certs_path()}
if config.flag_is_true('ARVADOS_API_HOST_INSECURE'):
ssl_options['cert_reqs'] = ssl.CERT_NONE
else:
ssl_options['cert_reqs'] = ssl.CERT_REQUIRED
# Warning: If the host part of url resolves to both IPv6 and
# IPv4 addresses (common with "localhost"), only one of them
# will be attempted -- and it might not be the right one. See
# ws4py's WebSocketBaseClient.__init__.
super(EventClient, self).__init__(url, ssl_options=ssl_options)
self.filters = filters
self.on_event = on_event
def __init__(self, url, filters, on_event, last_log_id):
ssl_options = {'ca_certs': arvados.util.ca_certs_path()}
if config.flag_is_true('ARVADOS_API_HOST_INSECURE'):
ssl_options['cert_reqs'] = ssl.CERT_NONE
else:
ssl_options['cert_reqs'] = ssl.CERT_REQUIRED
# Warning: If the host part of url resolves to both IPv6 and
# IPv4 addresses (common with "localhost"), only one of them
# will be attempted -- and it might not be the right one. See
# ws4py's WebSocketBaseClient.__init__.
super(EventClient, self).__init__(url, ssl_options=ssl_options)
self.filters = filters
self.on_event = on_event
self.stop = threading.Event()
self.last_log_id = last_log_id
def __init__(self, url, filters, on_event):
# Prefer system's CA certificates (if available)
ssl_options = {}
certs_path = '/etc/ssl/certs/ca-certificates.crt'
if os.path.exists(certs_path):
ssl_options['ca_certs'] = certs_path
if config.flag_is_true('ARVADOS_API_HOST_INSECURE'):
ssl_options['cert_reqs'] = ssl.CERT_NONE
else:
ssl_options['cert_reqs'] = ssl.CERT_REQUIRED
# Warning: If the host part of url resolves to both IPv6 and
# IPv4 addresses (common with "localhost"), only one of them
# will be attempted -- and it might not be the right one. See
# ws4py's WebSocketBaseClient.__init__.
super(EventClient, self).__init__(url, ssl_options=ssl_options)
self.filters = filters
self.on_event = on_event
def __init__(self, url, filters, on_event):
# Prefer system's CA certificates (if available)
ssl_options = {}
certs_path = '/etc/ssl/certs/ca-certificates.crt'
if os.path.exists(certs_path):
ssl_options['ca_certs'] = certs_path
if config.flag_is_true('ARVADOS_API_HOST_INSECURE'):
ssl_options['cert_reqs'] = ssl.CERT_NONE
else:
ssl_options['cert_reqs'] = ssl.CERT_REQUIRED
# Warning: If the host part of url resolves to both IPv6 and
# IPv4 addresses (common with "localhost"), only one of them
# will be attempted -- and it might not be the right one. See
# ws4py's WebSocketBaseClient.__init__.
super(EventClient, self).__init__(url, ssl_options=ssl_options)
self.filters = filters
self.on_event = on_event
def api(version=None, cache=True, host=None, token=None, insecure=False, **kwargs):
"""Return an apiclient Resources object for an Arvados instance.
Arguments:
* version: A string naming the version of the Arvados API to use (for
example, 'v1').
* cache: If True (default), return an existing Resources object if
one already exists with the same endpoint and credentials. If
False, create a new one, and do not keep it in the cache (i.e.,
do not return it from subsequent api(cache=True) calls with
matching endpoint and credentials).
* host: The Arvados API server host (and optional :port) to connect to.
* token: The authentication token to send with each API call.
* insecure: If True, ignore SSL certificate validation errors.
Additional keyword arguments will be passed directly to
`apiclient.discovery.build` if a new Resource object is created.
If the `discoveryServiceUrl` or `http` keyword arguments are
missing, this function will set default values for them, based on
the current Arvados configuration settings.
"""
if not version:
version = 'v1'
_logger.info("Using default API version. " +
"Call arvados.api('%s') instead." %
version)
if 'discoveryServiceUrl' in kwargs:
if host:
raise ValueError("both discoveryServiceUrl and host provided")
# Here we can't use a token from environment, config file,
# etc. Those probably have nothing to do with the host
# provided by the caller.
if not token:
raise ValueError("discoveryServiceUrl provided, but token missing")
elif host and token:
pass
elif not host and not token:
# Load from user configuration or environment
for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
if x not in config.settings():
raise ValueError("%s is not set. Aborting." % x)
host = config.get('ARVADOS_API_HOST')
token = config.get('ARVADOS_API_TOKEN')
insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE')
else:
# Caller provided one but not the other
if not host:
raise ValueError("token argument provided, but host missing.")
else:
raise ValueError("host argument provided, but token missing.")
if host:
# Caller wants us to build the discoveryServiceUrl
kwargs['discoveryServiceUrl'] = (
'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host,))
if cache:
connprofile = (version, host, token, insecure)
svc = conncache.get(connprofile)
if svc:
return svc
if 'http' not in kwargs:
http_kwargs = {}
# Prefer system's CA certificates (if available) over httplib2's.
certs_path = '/etc/ssl/certs/ca-certificates.crt'
if os.path.exists(certs_path):
http_kwargs['ca_certs'] = certs_path
if cache:
http_kwargs['cache'] = http_cache('discovery')
if insecure:
http_kwargs['disable_ssl_certificate_validation'] = True
kwargs['http'] = httplib2.Http(**http_kwargs)
credentials = CredentialsFromToken(api_token=token)
kwargs['http'] = credentials.authorize(kwargs['http'])
svc = apiclient.discovery.build('arvados', version, **kwargs)
svc.api_token = token
kwargs['http'].cache = None
if cache:
conncache[connprofile] = svc
return svc
def api(version=None,
cache=True,
host=None,
token=None,
insecure=False,
**kwargs):
"""Return an apiclient Resources object for an Arvados instance.
Arguments:
* version: A string naming the version of the Arvados API to use (for
example, 'v1').
* cache: If True (default), return an existing Resources object if
one already exists with the same endpoint and credentials. If
False, create a new one, and do not keep it in the cache (i.e.,
do not return it from subsequent api(cache=True) calls with
matching endpoint and credentials).
* host: The Arvados API server host (and optional :port) to connect to.
* token: The authentication token to send with each API call.
* insecure: If True, ignore SSL certificate validation errors.
Additional keyword arguments will be passed directly to
`apiclient.discovery.build` if a new Resource object is created.
If the `discoveryServiceUrl` or `http` keyword arguments are
missing, this function will set default values for them, based on
the current Arvados configuration settings.
"""
if not version:
version = 'v1'
_logger.info("Using default API version. " +
"Call arvados.api('%s') instead." % version)
if 'discoveryServiceUrl' in kwargs:
if host:
raise ValueError("both discoveryServiceUrl and host provided")
# Here we can't use a token from environment, config file,
# etc. Those probably have nothing to do with the host
# provided by the caller.
if not token:
raise ValueError("discoveryServiceUrl provided, but token missing")
elif host and token:
pass
elif not host and not token:
# Load from user configuration or environment
for x in ['ARVADOS_API_HOST', 'ARVADOS_API_TOKEN']:
if x not in config.settings():
raise ValueError("%s is not set. Aborting." % x)
host = config.get('ARVADOS_API_HOST')
token = config.get('ARVADOS_API_TOKEN')
insecure = config.flag_is_true('ARVADOS_API_HOST_INSECURE')
else:
# Caller provided one but not the other
if not host:
raise ValueError("token argument provided, but host missing.")
else:
raise ValueError("host argument provided, but token missing.")
if host:
# Caller wants us to build the discoveryServiceUrl
kwargs['discoveryServiceUrl'] = (
'https://%s/discovery/v1/apis/{api}/{apiVersion}/rest' % (host, ))
if cache:
connprofile = (version, host, token, insecure)
svc = conncache.get(connprofile)
if svc:
return svc
if 'http' not in kwargs:
http_kwargs = {}
# Prefer system's CA certificates (if available) over httplib2's.
certs_path = '/etc/ssl/certs/ca-certificates.crt'
if os.path.exists(certs_path):
http_kwargs['ca_certs'] = certs_path
if cache:
http_kwargs['cache'] = http_cache('discovery')
if insecure:
http_kwargs['disable_ssl_certificate_validation'] = True
kwargs['http'] = httplib2.Http(**http_kwargs)
credentials = CredentialsFromToken(api_token=token)
kwargs['http'] = credentials.authorize(kwargs['http'])
svc = apiclient.discovery.build('arvados', version, **kwargs)
svc.api_token = token
kwargs['http'].cache = None
if cache:
conncache[connprofile] = svc
return svc
