def global_client_object(cls):
global global_client_object
# Previously, KeepClient would change its behavior at runtime based
# on these configuration settings. We simulate that behavior here
# by checking the values and returning a new KeepClient if any of
# them have changed.
key = (config.get('ARVADOS_API_HOST'), config.get('ARVADOS_API_TOKEN'),
config.flag_is_true('ARVADOS_API_HOST_INSECURE'),
config.get('ARVADOS_KEEP_PROXY'),
config.get('ARVADOS_EXTERNAL_CLIENT') == 'true',
os.environ.get('KEEP_LOCAL_STORE'))
if (global_client_object is None) or (cls._last_key != key):
global_client_object = KeepClient()
cls._last_key = key
return global_client_object
def global_client_object(cls):
global global_client_object
# Previously, KeepClient would change its behavior at runtime based
# on these configuration settings. We simulate that behavior here
# by checking the values and returning a new KeepClient if any of
# them have changed.
key = (config.get('ARVADOS_API_HOST'),
config.get('ARVADOS_API_TOKEN'),
config.flag_is_true('ARVADOS_API_HOST_INSECURE'),
config.get('ARVADOS_KEEP_PROXY'),
config.get('ARVADOS_EXTERNAL_CLIENT') == 'true',
os.environ.get('KEEP_LOCAL_STORE'))
if (global_client_object is None) or (cls._last_key != key):
global_client_object = KeepClient()
cls._last_key = key
return global_client_object
def __init__(self, api_client=None, proxy=None,
timeout=DEFAULT_TIMEOUT, proxy_timeout=DEFAULT_PROXY_TIMEOUT,
api_token=None, local_store=None, block_cache=None,
num_retries=0, session=None):
"""Initialize a new KeepClient.
Arguments:
:api_client:
The API client to use to find Keep services. If not
provided, KeepClient will build one from available Arvados
configuration.
:proxy:
If specified, this KeepClient will send requests to this Keep
proxy. Otherwise, KeepClient will fall back to the setting of the
ARVADOS_KEEP_PROXY configuration setting. If you want to ensure
KeepClient does not use a proxy, pass in an empty string.
:timeout:
The initial timeout (in seconds) for HTTP requests to Keep
non-proxy servers. A tuple of three floats is interpreted as
(connection_timeout, read_timeout, minimum_bandwidth). A connection
will be aborted if the average traffic rate falls below
minimum_bandwidth bytes per second over an interval of read_timeout
seconds. Because timeouts are often a result of transient server
load, the actual connection timeout will be increased by a factor
of two on each retry.
Default: (2, 256, 32768).
:proxy_timeout:
The initial timeout (in seconds) for HTTP requests to
Keep proxies. A tuple of three floats is interpreted as
(connection_timeout, read_timeout, minimum_bandwidth). The behavior
described above for adjusting connection timeouts on retry also
applies.
Default: (20, 256, 32768).
:api_token:
If you're not using an API client, but only talking
directly to a Keep proxy, this parameter specifies an API token
to authenticate Keep requests. It is an error to specify both
api_client and api_token. If you specify neither, KeepClient
will use one available from the Arvados configuration.
:local_store:
If specified, this KeepClient will bypass Keep
services, and save data to the named directory. If unspecified,
KeepClient will fall back to the setting of the $KEEP_LOCAL_STORE
environment variable. If you want to ensure KeepClient does not
use local storage, pass in an empty string. This is primarily
intended to mock a server for testing.
:num_retries:
The default number of times to retry failed requests.
This will be used as the default num_retries value when get() and
put() are called. Default 0.
"""
self.lock = threading.Lock()
if proxy is None:
proxy = config.get('ARVADOS_KEEP_PROXY')
if api_token is None:
if api_client is None:
api_token = config.get('ARVADOS_API_TOKEN')
else:
api_token = api_client.api_token
elif api_client is not None:
raise ValueError(
"can't build KeepClient with both API client and token")
if local_store is None:
local_store = os.environ.get('KEEP_LOCAL_STORE')
self.block_cache = block_cache if block_cache else KeepBlockCache()
self.timeout = timeout
self.proxy_timeout = proxy_timeout
self._user_agent_pool = Queue.LifoQueue()
self.upload_counter = Counter()
self.download_counter = Counter()
self.put_counter = Counter()
self.get_counter = Counter()
self.hits_counter = Counter()
self.misses_counter = Counter()
if local_store:
self.local_store = local_store
self.get = self.local_store_get
self.put = self.local_store_put
else:
self.num_retries = num_retries
self.max_replicas_per_service = None
if proxy:
if not proxy.endswith('/'):
proxy += '/'
self.api_token = api_token
self._gateway_services = {}
self._keep_services = [{
'uuid': 'proxy',
'service_type': 'proxy',
'_service_root': proxy,
}]
self._writable_services = self._keep_services
self.using_proxy = True
self._static_services_list = True
else:
# It's important to avoid instantiating an API client
# unless we actually need one, for testing's sake.
if api_client is None:
api_client = arvados.api('v1')
self.api_client = api_client
self.api_token = api_client.api_token
self._gateway_services = {}
self._keep_services = None
self._writable_services = None
self.using_proxy = None
self._static_services_list = False
def __init__(self,
api_client=None,
proxy=None,
timeout=DEFAULT_TIMEOUT,
proxy_timeout=DEFAULT_PROXY_TIMEOUT,
api_token=None,
local_store=None,
block_cache=None,
num_retries=0,
session=None):
"""Initialize a new KeepClient.
Arguments:
:api_client:
The API client to use to find Keep services. If not
provided, KeepClient will build one from available Arvados
configuration.
:proxy:
If specified, this KeepClient will send requests to this Keep
proxy. Otherwise, KeepClient will fall back to the setting of the
ARVADOS_KEEP_SERVICES or ARVADOS_KEEP_PROXY configuration settings.
If you want to KeepClient does not use a proxy, pass in an empty
string.
:timeout:
The initial timeout (in seconds) for HTTP requests to Keep
non-proxy servers. A tuple of three floats is interpreted as
(connection_timeout, read_timeout, minimum_bandwidth). A connection
will be aborted if the average traffic rate falls below
minimum_bandwidth bytes per second over an interval of read_timeout
seconds. Because timeouts are often a result of transient server
load, the actual connection timeout will be increased by a factor
of two on each retry.
Default: (2, 256, 32768).
:proxy_timeout:
The initial timeout (in seconds) for HTTP requests to
Keep proxies. A tuple of three floats is interpreted as
(connection_timeout, read_timeout, minimum_bandwidth). The behavior
described above for adjusting connection timeouts on retry also
applies.
Default: (20, 256, 32768).
:api_token:
If you're not using an API client, but only talking
directly to a Keep proxy, this parameter specifies an API token
to authenticate Keep requests. It is an error to specify both
api_client and api_token. If you specify neither, KeepClient
will use one available from the Arvados configuration.
:local_store:
If specified, this KeepClient will bypass Keep
services, and save data to the named directory. If unspecified,
KeepClient will fall back to the setting of the $KEEP_LOCAL_STORE
environment variable. If you want to ensure KeepClient does not
use local storage, pass in an empty string. This is primarily
intended to mock a server for testing.
:num_retries:
The default number of times to retry failed requests.
This will be used as the default num_retries value when get() and
put() are called. Default 0.
"""
self.lock = threading.Lock()
if proxy is None:
if config.get('ARVADOS_KEEP_SERVICES'):
proxy = config.get('ARVADOS_KEEP_SERVICES')
else:
proxy = config.get('ARVADOS_KEEP_PROXY')
if api_token is None:
if api_client is None:
api_token = config.get('ARVADOS_API_TOKEN')
else:
api_token = api_client.api_token
elif api_client is not None:
raise ValueError(
"can't build KeepClient with both API client and token")
if local_store is None:
local_store = os.environ.get('KEEP_LOCAL_STORE')
self.block_cache = block_cache if block_cache else KeepBlockCache()
self.timeout = timeout
self.proxy_timeout = proxy_timeout
self._user_agent_pool = queue.LifoQueue()
self.upload_counter = Counter()
self.download_counter = Counter()
self.put_counter = Counter()
self.get_counter = Counter()
self.hits_counter = Counter()
self.misses_counter = Counter()
if local_store:
self.local_store = local_store
self.get = self.local_store_get
self.put = self.local_store_put
else:
self.num_retries = num_retries
self.max_replicas_per_service = None
if proxy:
proxy_uris = proxy.split()
for i in range(len(proxy_uris)):
if not proxy_uris[i].endswith('/'):
proxy_uris[i] += '/'
# URL validation
url = urllib.parse.urlparse(proxy_uris[i])
if not (url.scheme and url.netloc):
raise arvados.errors.ArgumentError(
"Invalid proxy URI: {}".format(proxy_uris[i]))
self.api_token = api_token
self._gateway_services = {}
self._keep_services = [{
'uuid': "00000-bi6l4-%015d" % idx,
'service_type': 'proxy',
'_service_root': uri,
} for idx, uri in enumerate(proxy_uris)]
self._writable_services = self._keep_services
self.using_proxy = True
self._static_services_list = True
else:
# It's important to avoid instantiating an API client
# unless we actually need one, for testing's sake.
if api_client is None:
api_client = arvados.api('v1')
self.api_client = api_client
self.api_token = api_client.api_token
self._gateway_services = {}
self._keep_services = None
self._writable_services = None
self.using_proxy = None
self._static_services_list = False
def __init__(self,
api_client=None,
proxy=None,
timeout=300,
api_token=None,
local_store=None,
block_cache=None,
num_retries=0):
"""Initialize a new KeepClient.
Arguments:
* api_client: The API client to use to find Keep services. If not
provided, KeepClient will build one from available Arvados
configuration.
* proxy: If specified, this KeepClient will send requests to this
Keep proxy. Otherwise, KeepClient will fall back to the setting
of the ARVADOS_KEEP_PROXY configuration setting. If you want to
ensure KeepClient does not use a proxy, pass in an empty string.
* timeout: The timeout for all HTTP requests, in seconds. Default
300.
* api_token: If you're not using an API client, but only talking
directly to a Keep proxy, this parameter specifies an API token
to authenticate Keep requests. It is an error to specify both
api_client and api_token. If you specify neither, KeepClient
will use one available from the Arvados configuration.
* local_store: If specified, this KeepClient will bypass Keep
services, and save data to the named directory. If unspecified,
KeepClient will fall back to the setting of the $KEEP_LOCAL_STORE
environment variable. If you want to ensure KeepClient does not
use local storage, pass in an empty string. This is primarily
intended to mock a server for testing.
* num_retries: The default number of times to retry failed requests.
This will be used as the default num_retries value when get() and
put() are called. Default 0.
"""
self.lock = threading.Lock()
if proxy is None:
proxy = config.get('ARVADOS_KEEP_PROXY')
if api_token is None:
api_token = config.get('ARVADOS_API_TOKEN')
elif api_client is not None:
raise ValueError(
"can't build KeepClient with both API client and token")
if local_store is None:
local_store = os.environ.get('KEEP_LOCAL_STORE')
self.block_cache = block_cache if block_cache else KeepBlockCache()
if local_store:
self.local_store = local_store
self.get = self.local_store_get
self.put = self.local_store_put
else:
self.timeout = timeout
self.num_retries = num_retries
if proxy:
if not proxy.endswith('/'):
proxy += '/'
self.api_token = api_token
self.service_roots = [proxy]
self.using_proxy = True
self.static_service_roots = True
else:
# It's important to avoid instantiating an API client
# unless we actually need one, for testing's sake.
if api_client is None:
api_client = arvados.api('v1')
self.api_client = api_client
self.api_token = api_client.api_token
self.service_roots = None
self.using_proxy = None
self.static_service_roots = False
def __init__(self, api_client=None, proxy=None,
timeout=DEFAULT_TIMEOUT, proxy_timeout=DEFAULT_PROXY_TIMEOUT,
api_token=None, local_store=None, block_cache=None,
num_retries=0, session=None):
"""Initialize a new KeepClient.
Arguments:
:api_client:
The API client to use to find Keep services. If not
provided, KeepClient will build one from available Arvados
configuration.
:proxy:
If specified, this KeepClient will send requests to this Keep
proxy. Otherwise, KeepClient will fall back to the setting of the
ARVADOS_KEEP_PROXY configuration setting. If you want to ensure
KeepClient does not use a proxy, pass in an empty string.
:timeout:
The initial timeout (in seconds) for HTTP requests to Keep
non-proxy servers. A tuple of two floats is interpreted as
(connection_timeout, read_timeout): see
http://docs.python-requests.org/en/latest/user/advanced/#timeouts.
Because timeouts are often a result of transient server load, the
actual connection timeout will be increased by a factor of two on
each retry.
Default: (2, 300).
:proxy_timeout:
The initial timeout (in seconds) for HTTP requests to
Keep proxies. A tuple of two floats is interpreted as
(connection_timeout, read_timeout). The behavior described
above for adjusting connection timeouts on retry also applies.
Default: (20, 300).
:api_token:
If you're not using an API client, but only talking
directly to a Keep proxy, this parameter specifies an API token
to authenticate Keep requests. It is an error to specify both
api_client and api_token. If you specify neither, KeepClient
will use one available from the Arvados configuration.
:local_store:
If specified, this KeepClient will bypass Keep
services, and save data to the named directory. If unspecified,
KeepClient will fall back to the setting of the $KEEP_LOCAL_STORE
environment variable. If you want to ensure KeepClient does not
use local storage, pass in an empty string. This is primarily
intended to mock a server for testing.
:num_retries:
The default number of times to retry failed requests.
This will be used as the default num_retries value when get() and
put() are called. Default 0.
:session:
The requests.Session object to use for get() and put() requests.
Will create one if not specified.
"""
self.lock = threading.Lock()
if proxy is None:
proxy = config.get('ARVADOS_KEEP_PROXY')
if api_token is None:
if api_client is None:
api_token = config.get('ARVADOS_API_TOKEN')
else:
api_token = api_client.api_token
elif api_client is not None:
raise ValueError(
"can't build KeepClient with both API client and token")
if local_store is None:
local_store = os.environ.get('KEEP_LOCAL_STORE')
self.block_cache = block_cache if block_cache else KeepBlockCache()
self.timeout = timeout
self.proxy_timeout = proxy_timeout
if local_store:
self.local_store = local_store
self.get = self.local_store_get
self.put = self.local_store_put
else:
self.num_retries = num_retries
self.session = session if session is not None else requests.Session()
if proxy:
if not proxy.endswith('/'):
proxy += '/'
self.api_token = api_token
self._gateway_services = {}
self._keep_services = [{
'uuid': 'proxy',
'_service_root': proxy,
}]
self.using_proxy = True
self._static_services_list = True
else:
# It's important to avoid instantiating an API client
# unless we actually need one, for testing's sake.
if api_client is None:
api_client = arvados.api('v1')
self.api_client = api_client
self.api_token = api_client.api_token
self._gateway_services = {}
self._keep_services = None
self.using_proxy = None
self._static_services_list = False
