def __init__(self):
endpoint = config.online_swap.caesar.panoptes_endpoint
url = config.online_swap.address._panoptes
endpoint = url % {'endpoint': endpoint}
self.client = Panoptes(endpoint=endpoint)
self.lock = threading.Lock()
def setUp(self):
self.http_result = Mock()
self.client = Panoptes()
self.client.valid_bearer_token = Mock()
self.client.valid_bearer_token.return_value = True
self.client.bearer_token = '1234'
self.client.session = Mock()
self.client.session.request = Mock()
self.client.session.request.return_value = self.http_result
def test_early(self):
target = datetime.datetime(2017, 1, 1, 10, 0, 0)
MockDate.fake(target)
client = Panoptes()
client.bearer_token = True
client.bearer_expires = datetime.datetime(2017, 1, 1, 12, 0, 0)
assert client.valid_bearer_token() is True
def test_in_buffer(self):
target = datetime.datetime(2017, 1, 1, 11, 59, 0)
MockDate.fake(target)
client = Panoptes()
client.bearer_token = True
client.bearer_expires = datetime.datetime(2017, 1, 1, 12, 0, 0)
assert client.valid_bearer_token() is False
def test_early(self):
target = datetime.datetime(2017, 1, 1, 10, 0, 0)
MockDate.fake(target)
client = Panoptes()
client.bearer_token = True
client.bearer_expires = datetime.datetime(2017, 1, 1, 12, 0, 0)
assert client.valid_bearer_token() is True
def test_in_buffer(self):
target = datetime.datetime(2017, 1, 1, 11, 59, 0)
MockDate.fake(target)
client = Panoptes()
client.bearer_token = True
client.bearer_expires = datetime.datetime(2017, 1, 1, 12, 0, 0)
assert client.valid_bearer_token() is False
def http_get(cls, path, params={}, headers={}):
workflow = params.pop('workflow')
return Panoptes.client().get(
Workflow.url(workflow.id) + cls.url(path),
params,
headers,
)
class _AuthCaesar:
def __init__(self):
endpoint = config.online_swap.caesar.panoptes_endpoint
url = config.online_swap.address._panoptes
endpoint = url % {'endpoint': endpoint}
self.client = Panoptes(endpoint=endpoint)
self.lock = threading.Lock()
@property
def token(self):
return self.client.get_bearer_token()
def login(self):
with self.lock:
logger.info('Logging in to panoptes')
user = input('Username: '******'adding authorization header')
with self.lock:
if self.token is None:
raise self.NotLoggedIn
token = self.token
if headers is None:
headers = {}
headers.update({'Authorization': 'Bearer %s' % token})
return headers
class NotLoggedIn(Exception):
def __init__(self):
super().__init__(
'Need to log in first. Either set panoptes oauth2 bearer '
'token in config.online_swap.caesar.OAUTH, or try running '
'app again with --login flag')
def avatar_get(cls, path, params={}, headers={}):
project = params.pop('project')
avatar_response = Panoptes.client().get(
Project.url(project.id) + cls.url(path),
params,
headers,
)
return avatar_response
def save(self, client=None):
"""
Like :py:meth:`.PanoptesObject.save`, but also uploads any local files
which have previosly been added to the subject with
:py:meth:`add_location`. Automatically retries uploads on error.
If multiple local files are to be uploaded, several files will be
uploaded simultaneously to save time.
"""
if not client:
client = Panoptes.client()
async_save = hasattr(self._local, 'save_exec')
with client:
if async_save:
try:
# The recursive call will exec in a new thread, so
# self._local.save_exec will be undefined above
self._async_future = self._local.save_exec.submit(
self.save,
client=client,
)
return
except RuntimeError:
del self._local.save_exec
async_save = False
if not self.metadata == self._original_metadata:
self.modified_attributes.add('metadata')
response = retry(
super(Subject, self).save,
attempts=UPLOAD_RETRY_LIMIT,
sleeptime=RETRY_BACKOFF_INTERVAL,
retry_exceptions=(PanoptesAPIException, ),
log_args=False,
)
if not response:
return
try:
if async_save:
upload_exec = self._local.save_exec
else:
upload_exec = ThreadPoolExecutor(
max_workers=ASYNC_SAVE_THREADS, )
for location, media_data in zip(
response['subjects'][0]['locations'],
self._media_files):
if not media_data:
continue
for media_type, url in location.items():
upload_exec.submit(
retry,
self._upload_media,
args=(url, media_data, media_type),
attempts=UPLOAD_RETRY_LIMIT,
sleeptime=RETRY_BACKOFF_INTERVAL,
retry_exceptions=(
requests.exceptions.RequestException, ),
log_args=False,
)
finally:
if not async_save:
upload_exec.shutdown()
class _AuthCaesar:
"""
Singleton instance to manage how SWAP authenticate with Caesar.
Uses the Panoptes python client to manage OAuth2 auth. A user
needs to enter their Zooniverse once on startup. From then on
the instance uses a Bearer token to authenticate with Caesar, and
can refresh the token periodically when it expires.
"""
def __init__(self):
endpoint = config.online_swap.caesar.panoptes_endpoint
url = config.online_swap.address._panoptes
endpoint = url % {'endpoint': endpoint}
self.client = Panoptes(endpoint=endpoint)
self.lock = threading.Lock()
@property
def token(self):
"""
Fetch the bearer token from the panoptes python client
"""
return self.client.get_bearer_token()
def login(self):
"""
Prompt the user for their Zooniverse username and password
and send them to the panoptes python client to log in
"""
with self.lock:
logger.info('Logging in to panoptes')
user = input('Username: '******'adding authorization header')
with self.lock:
if self.token is None:
raise self.NotLoggedIn
token = self.token
if headers is None:
headers = {}
headers.update({
'Authorization': 'Bearer %s' % token
})
return headers
class NotLoggedIn(Exception):
"""
Raised if the panoptes client is not logged in
"""
def __init__(self):
super().__init__(
'Need to log in first. Try running again with '
'the --login flag')
def client(cls):
if cls._client is None:
cls._client = Panoptes(login='******')
return cls._client
def test_has_no_token(self):
client = Panoptes()
assert client.has_bearer_token() is False
class TestRetries(unittest.TestCase):
def setUp(self):
self.http_result = Mock()
self.client = Panoptes()
self.client.valid_bearer_token = Mock()
self.client.valid_bearer_token.return_value = True
self.client.bearer_token = '1234'
self.client.session = Mock()
self.client.session.request = Mock()
self.client.session.request.return_value = self.http_result
def assert_retry(self, *args, **kwargs):
self.assertTrue(kwargs.get('retry', False))
result = Mock()
result.status_code = 204
return result
def assert_no_retry(self, *args, **kwargs):
self.assertFalse(kwargs.get('retry', True))
result = Mock()
result.status_code = 204
return result
@patch('panoptes_client.panoptes.RETRY_BACKOFF_INTERVAL', 1)
def test_request_retry_success(self):
self.http_result.status_code = 200
self.assertEqual(
self.client.http_request('GET', '', retry=True),
self.http_result,
)
self.assertEqual(
self.client.session.request.call_count,
1,
)
@patch('panoptes_client.panoptes.RETRY_BACKOFF_INTERVAL', 1)
def test_request_retry_no_success(self):
self.http_result.status_code = 500
with self.assertRaises(PanoptesAPIException):
self.assertEqual(
self.client.http_request('GET', '', retry=True),
self.http_result,
)
self.assertEqual(
self.client.session.request.call_count,
HTTP_RETRY_LIMIT,
)
@patch('panoptes_client.panoptes.RETRY_BACKOFF_INTERVAL', 1)
def test_request_no_retry_success(self):
self.http_result.status_code = 200
self.assertEqual(
self.client.http_request('GET', '', retry=False),
self.http_result,
)
self.assertEqual(
self.client.session.request.call_count,
1,
)
@patch('panoptes_client.panoptes.RETRY_BACKOFF_INTERVAL', 1)
def test_request_no_retry_no_success(self):
self.http_result.status_code = 500
with self.assertRaises(PanoptesAPIException):
self.assertEqual(
self.client.http_request('GET', '', retry=False),
self.http_result,
)
self.assertEqual(
self.client.session.request.call_count,
1,
)
def test_json_retry(self):
self.client.http_request = self.assert_retry
self.client.json_request('', '', retry=True)
def test_json_no_retry(self):
self.client.http_request = self.assert_no_retry
self.client.json_request('', '', retry=False)
def test_get_retry(self):
self.client.json_request = self.assert_retry
self.client.get('', retry=True)
def test_get_no_retry(self):
self.client.json_request = self.assert_no_retry
self.client.get('', retry=False)
def test_get_request_retry(self):
self.client.http_request = self.assert_retry
self.client.get_request('', retry=True)
def test_get_request_no_retry(self):
self.client.http_request = self.assert_no_retry
self.client.get_request('', retry=False)
def test_put_retry(self):
self.client.json_request = self.assert_retry
self.client.put('', retry=True)
def test_put_no_retry(self):
self.client.json_request = self.assert_no_retry
self.client.put('', retry=False)
def test_put_request_retry(self):
self.client.http_request = self.assert_retry
self.client.put_request('', retry=True)
def test_put_request_no_retry(self):
self.client.http_request = self.assert_no_retry
self.client.put_request('', retry=False)
def test_post_retry(self):
self.client.json_request = self.assert_retry
self.client.post('', retry=True)
def test_post_no_retry(self):
self.client.json_request = self.assert_no_retry
self.client.post('', retry=False)
def test_post_request_retry(self):
self.client.http_request = self.assert_retry
self.client.post_request('', retry=True)
def test_post_request_no_retry(self):
self.client.http_request = self.assert_no_retry
self.client.post_request('', retry=False)
def test_delete_retry(self):
self.client.json_request = self.assert_retry
self.client.delete('', retry=True)
def test_delete_no_retry(self):
self.client.json_request = self.assert_no_retry
self.client.delete('', retry=False)
def test_delete_request_retry(self):
self.client.http_request = self.assert_retry
self.client.delete_request('', retry=True)
def test_delete_request_no_retry(self):
self.client.http_request = self.assert_no_retry
self.client.delete_request('', retry=False)
def test_has_no_token(self):
client = Panoptes()
assert client.has_bearer_token() is False
def test_has_token(self):
client = Panoptes()
client.bearer_token = True
assert client.has_bearer_token() is True
def test_has_token(self):
client = Panoptes()
client.bearer_token = True
assert client.has_bearer_token() is True
def save(self, client=None):
"""
Like :py:meth:`.PanoptesObject.save`, but also uploads any local files
which have previosly been added to the subject with
:py:meth:`add_location`. Automatically retries uploads on error.
If multiple local files are to be uploaded, several files will be
uploaded simultaneously to save time.
"""
if not client:
client = Panoptes.client()
async_save = hasattr(self._local, 'save_exec')
with client:
if async_save:
try:
# The recursive call will exec in a new thread, so
# self._local.save_exec will be undefined above
self._async_future = self._local.save_exec.submit(
self.save,
client=client,
)
return
except RuntimeError:
del self._local.save_exec
async_save = False
if not self.metadata == self._original_metadata:
self.modified_attributes.add('metadata')
response = retry(
super(Subject, self).save,
attempts=UPLOAD_RETRY_LIMIT,
sleeptime=RETRY_BACKOFF_INTERVAL,
retry_exceptions=(PanoptesAPIException,),
log_args=False,
)
if not response:
return
try:
if async_save:
upload_exec = self._local.save_exec
else:
upload_exec = ThreadPoolExecutor(
max_workers=ASYNC_SAVE_THREADS,
)
for location, media_data in zip(
response['subjects'][0]['locations'],
self._media_files
):
if not media_data:
continue
for media_type, url in location.items():
upload_exec.submit(
retry,
self._upload_media,
args=(url, media_data, media_type),
attempts=UPLOAD_RETRY_LIMIT,
sleeptime=RETRY_BACKOFF_INTERVAL,
retry_exceptions=(
requests.exceptions.RequestException,
),
log_args=False,
)
finally:
if not async_save:
upload_exec.shutdown()