def health_check(client, **kwargs):
"""
This function calls client.cluster.health and, based on the args provided,
will return `True` or `False` depending on whether that particular keyword
appears in the output, and has the expected value.
If multiple keys are provided, all must match for a `True` response.
:arg client: An :class:`elasticsearch.Elasticsearch` client object
"""
logger.debug('KWARGS= "{0}"'.format(kwargs))
klist = list(kwargs.keys())
if len(klist) < 1:
raise MissingArgument('Must provide at least one keyword argument')
hc_data = client.cluster.health()
response = True
for k in klist:
# First, verify that all kwargs are in the list
if not k in list(hc_data.keys()):
raise ConfigurationError('Key "{0}" not in cluster health output')
if not hc_data[k] == kwargs[k]:
logger.debug('NO MATCH: Value for key "{0}", health check data: '
'{1}'.format(kwargs[k], hc_data[k]))
response = False
else:
logger.debug('MATCH: Value for key "{0}", health check data: '
'{1}'.format(kwargs[k], hc_data[k]))
if response:
logger.info('Health Check for all provided keys passed.')
return response
def safe_to_snap(client, repository=None, retry_interval=120, retry_count=3):
"""
Ensure there are no snapshots in progress. Pause and retry accordingly
:arg client: An :class:`elasticsearch.Elasticsearch` client object
:arg repository: The Elasticsearch snapshot repository to use
:arg retry_interval: Number of seconds to delay betwen retries. Default:
120 (seconds)
:arg retry_count: Number of attempts to make. Default: 3
:rtype: bool
"""
if not repository:
raise MissingArgument('No value for "repository" provided')
for count in range(1, retry_count+1):
in_progress = snapshot_in_progress(
client, repository=repository
)
ongoing_task = find_snapshot_tasks(client)
if in_progress or ongoing_task:
if in_progress:
logger.info(
'Snapshot already in progress: {0}'.format(in_progress))
elif ongoing_task:
logger.info('Snapshot activity detected in Tasks API')
logger.info(
'Pausing {0} seconds before retrying...'.format(retry_interval))
sleep(retry_interval)
logger.info('Retry {0} of {1}'.format(count, retry_count))
else:
return True
return False
def get_snapshot_data(client, repository=None):
"""
Get ``_all`` snapshots from repository and return a list.
:arg client: An :class:`elasticsearch.Elasticsearch` client object
:arg repository: The Elasticsearch snapshot repository to use
:rtype: list
"""
if not repository:
raise MissingArgument('No value for "repository" provided')
try:
return client.snapshot.get(
repository=repository, snapshot="_all")['snapshots']
except (TransportError, NotFoundError) as e:
raise FailedExecution(
'Unable to get snapshot information from repository: {0}. '
'Error: {1}'.format(repository, e)
)
def get_snapshot(client, repository=None, snapshot=''):
"""
Return information about a snapshot (or a comma-separated list of snapshots)
If no snapshot specified, it will return all snapshots. If none exist, an
empty dictionary will be returned.
:arg client: An :class:`elasticsearch.Elasticsearch` client object
:arg repository: The Elasticsearch snapshot repository to use
:arg snapshot: The snapshot name, or a comma-separated list of snapshots
:rtype: dict
"""
if not repository:
raise MissingArgument('No value for "repository" provided')
snapname = '_all' if snapshot == '' else snapshot
try:
return client.snapshot.get(repository=repository, snapshot=snapname)
except (TransportError, NotFoundError) as e:
raise FailedExecution(
'Unable to get information about snapshot {0} from repository: '
'{1}. Error: {2}'.format(snapname, repository, e)
)
def repository_exists(client, repository=None):
"""
Verify the existence of a repository
:arg client: An :class:`elasticsearch.Elasticsearch` client object
:arg repository: The Elasticsearch snapshot repository to use
:rtype: bool
"""
if not repository:
raise MissingArgument('No value for "repository" provided')
try:
test_result = get_repository(client, repository)
if repository in test_result:
logger.debug("Repository {0} exists.".format(repository))
return True
else:
logger.debug("Repository {0} not found...".format(repository))
return False
except Exception as e:
logger.debug(
'Unable to find repository "{0}": Error: '
'{1}'.format(repository, e)
)
return False
def create_repo_body(repo_type=None,
compress=True, chunk_size=None,
max_restore_bytes_per_sec=None,
max_snapshot_bytes_per_sec=None,
location=None,
bucket=None, region=None, base_path=None, access_key=None,
secret_key=None, **kwargs):
"""
Build the 'body' portion for use in creating a repository.
:arg repo_type: The type of repository (presently only `fs` and `s3`)
:arg compress: Turn on compression of the snapshot files. Compression is
applied only to metadata files (index mapping and settings). Data files
are not compressed. (Default: `True`)
:arg chunk_size: The chunk size can be specified in bytes or by using size
value notation, i.e. 1g, 10m, 5k. Defaults to `null` (unlimited chunk
size).
:arg max_restore_bytes_per_sec: Throttles per node restore rate. Defaults
to ``20mb`` per second.
:arg max_snapshot_bytes_per_sec: Throttles per node snapshot rate. Defaults
to ``20mb`` per second.
:arg location: Location of the snapshots. Required.
:arg bucket: `S3 only.` The name of the bucket to be used for snapshots.
Required.
:arg region: `S3 only.` The region where bucket is located. Defaults to
`US Standard`
:arg base_path: `S3 only.` Specifies the path within bucket to repository
data. Defaults to value of ``repositories.s3.base_path`` or to root
directory if not set.
:arg access_key: `S3 only.` The access key to use for authentication.
Defaults to value of ``cloud.aws.access_key``.
:arg secret_key: `S3 only.` The secret key to use for authentication.
Defaults to value of ``cloud.aws.secret_key``.
:returns: A dictionary suitable for creating a repository from the provided
arguments.
:rtype: dict
"""
# This shouldn't happen, but just in case...
if not repo_type:
raise MissingArgument('Missing required parameter --repo_type')
argdict = locals()
body = {}
body['type'] = argdict['repo_type']
body['settings'] = {}
settingz = [] # Differentiate from module settings
maybes = [
'compress', 'chunk_size',
'max_restore_bytes_per_sec', 'max_snapshot_bytes_per_sec'
]
s3 = ['bucket', 'region', 'base_path', 'access_key', 'secret_key']
settingz += [i for i in maybes if argdict[i]]
# Type 'fs'
if argdict['repo_type'] == 'fs':
settingz.append('location')
# Type 's3'
if argdict['repo_type'] == 's3':
settingz += [i for i in s3 if argdict[i]]
for k in settingz:
body['settings'][k] = argdict[k]
return body
def create_repository(client, **kwargs):
"""
Create repository with repository and body settings
:arg client: An :class:`elasticsearch.Elasticsearch` client object
:arg repository: The Elasticsearch snapshot repository to use
:arg repo_type: The type of repository (presently only `fs` and `s3`)
:arg compress: Turn on compression of the snapshot files. Compression is
applied only to metadata files (index mapping and settings). Data files
are not compressed. (Default: `True`)
:arg chunk_size: The chunk size can be specified in bytes or by using size
value notation, i.e. 1g, 10m, 5k. Defaults to `null` (unlimited chunk
size).
:arg max_restore_bytes_per_sec: Throttles per node restore rate. Defaults
to ``20mb`` per second.
:arg max_snapshot_bytes_per_sec: Throttles per node snapshot rate. Defaults
to ``20mb`` per second.
:arg location: Location of the snapshots. Required.
:arg bucket: `S3 only.` The name of the bucket to be used for snapshots.
Required.
:arg region: `S3 only.` The region where bucket is located. Defaults to
`US Standard`
:arg base_path: `S3 only.` Specifies the path within bucket to repository
data. Defaults to value of ``repositories.s3.base_path`` or to root
directory if not set.
:arg access_key: `S3 only.` The access key to use for authentication.
Defaults to value of ``cloud.aws.access_key``.
:arg secret_key: `S3 only.` The secret key to use for authentication.
Defaults to value of ``cloud.aws.secret_key``.
:arg skip_repo_fs_check: Skip verifying the repo after creation.
:returns: A boolean value indicating success or failure.
:rtype: bool
"""
if not 'repository' in kwargs:
raise MissingArgument('Missing required parameter "repository"')
else:
repository = kwargs['repository']
skip_repo_fs_check = kwargs.pop('skip_repo_fs_check', False)
params = {'verify': 'false' if skip_repo_fs_check else 'true'}
try:
body = create_repo_body(**kwargs)
logger.debug(
'Checking if repository {0} already exists...'.format(repository)
)
result = repository_exists(client, repository=repository)
logger.debug("Result = {0}".format(result))
if not result:
logger.debug(
'Repository {0} not in Elasticsearch. Continuing...'.format(
repository
)
)
client.snapshot.create_repository(repository=repository, body=body, params=params)
else:
raise FailedExecution(
'Unable to create repository {0}. A repository with that name '
'already exists.'.format(repository)
)
except TransportError as e:
raise FailedExecution(
"""
Unable to create repository {0}. Response Code: {1}. Error: {2}.
Check curator and elasticsearch logs for more information.
""".format(
repository, e.status_code, e.error
)
)
logger.debug("Repository {0} creation initiated...".format(repository))
return True
def wait_for_it(client,
action,
task_id=None,
snapshot=None,
repository=None,
index=None,
index_list=None,
wait_interval=9,
max_wait=-1):
"""
This function becomes one place to do all wait_for_completion type behaviors
:arg client: An :class:`elasticsearch.Elasticsearch` client object
:arg action: The action name that will identify how to wait
:arg task_id: If the action provided a task_id, this is where it must be
declared.
:arg snapshot: The name of the snapshot.
:arg repository: The Elasticsearch snapshot repository to use
:arg wait_interval: How frequently the specified "wait" behavior will be
polled to check for completion.
:arg max_wait: Number of seconds will the "wait" behavior persist
before giving up and raising an Exception. The default is -1, meaning
it will try forever.
"""
action_map = {
'allocation': {
'function': health_check,
'args': {
'relocating_shards': 0
},
},
'replicas': {
'function': health_check,
'args': {
'status': 'green'
},
},
'cluster_routing': {
'function': health_check,
'args': {
'relocating_shards': 0
},
},
'snapshot': {
'function': snapshot_check,
'args': {
'snapshot': snapshot,
'repository': repository
},
},
'restore': {
'function': restore_check,
'args': {
'index_list': index_list
},
},
'reindex': {
'function': task_check,
'args': {
'task_id': task_id
},
},
'shrink': {
'function': health_check,
'args': {
'status': 'green'
},
},
'relocate': {
'function': relocate_check,
'args': {
'index': index
}
},
}
wait_actions = list(action_map.keys())
if action not in wait_actions:
raise ConfigurationError(
'"action" must be one of {0}'.format(wait_actions))
if action == 'reindex' and task_id == None:
raise MissingArgument(
'A task_id must accompany "action" {0}'.format(action))
if action == 'snapshot' and ((snapshot == None) or (repository == None)):
raise MissingArgument(
'A snapshot and repository must accompany "action" {0}. snapshot: '
'{1}, repository: {2}'.format(action, snapshot, repository))
if action == 'restore' and index_list == None:
raise MissingArgument(
'An index_list must accompany "action" {0}'.format(action))
elif action == 'reindex':
try:
_ = client.tasks.get(task_id=task_id)
except Exception as e:
# This exception should only exist in API usage. It should never
# occur in regular Curator usage.
raise CuratorException(
'Unable to find task_id {0}. Exception: {1}'.format(
task_id, e))
# Now with this mapped, we can perform the wait as indicated.
start_time = datetime.now()
result = False
while True:
elapsed = int((datetime.now() - start_time).total_seconds())
logger.debug('Elapsed time: {0} seconds'.format(elapsed))
response = action_map[action]['function'](client,
**action_map[action]['args'])
logger.debug('Response: {0}'.format(response))
# Success
if response:
logger.debug(
'Action "{0}" finished executing (may or may not have been '
'successful)'.format(action))
result = True
break
# Not success, and reached maximum wait (if defined)
elif (max_wait != -1) and (elapsed >= max_wait):
logger.error(
'Unable to complete action "{0}" within max_wait ({1}) '
'seconds.'.format(action, max_wait))
break
# Not success, so we wait.
else:
logger.debug(
'Action "{0}" not yet complete, {1} total seconds elapsed. '
'Waiting {2} seconds before checking '
'again.'.format(action, elapsed, wait_interval))
time.sleep(wait_interval)
logger.debug('Result: {0}'.format(result))
if result == False:
raise ActionTimeout(
'Action "{0}" failed to complete in the max_wait period of '
'{1} seconds'.format(action, max_wait))