def get_deltas(annotations: Dict, deltas_annotation_key: str) -> List[timedelta]:
"""
Helper annotation-deltas-getter
Parameters
----------
annotations
Returns
-------
"""
try:
deltas_str = annotations[deltas_annotation_key]
except KeyError as exc:
raise AnnotationNotFound(
'No such annotation key',
key=deltas_annotation_key
) from exc
if not deltas_str:
raise AnnotationError('Invalid delta string', deltas_str=deltas_str)
try:
deltas = parse_deltas(deltas_str)
except DeltasParseError as exc:
raise AnnotationError(
'Invalid delta string',
deltas_str=deltas_str
) from exc
if deltas is None or not deltas:
raise AnnotationError(
'parse_deltas returned invalid deltas',
deltas_str=deltas_str,
deltas=deltas,
)
return deltas
async def rule_from_snapshotrule(
ctx: Context,
resource: SnapshotRule
) -> Optional[Rule]:
"""This tries to build a rule within a `SnapshotRule` resource -
the resource that we custom designed for this purpose.
This is invoked whenever Kubernetes tells us that such a resource
was created, deleted, or updated.
There are two separate ways a `SnapshotRule` can be used:
- A `SnapshotRule` resource can refer to a specific Cloud disk
id to be snapshotted, e.g. 'example-disk' on 'gcloud'. This
skips Kubernetes entirely.
- A `SnapshotRule` resource can refer to a `PersistentVolumeClaim`.
The disk this claim is bound to is the one we will snapshot.
Rather than defining the snapshot interval etc. as annotations
of the claim, they are defined here, in a separate resource.
"""
_log = _logger.new(resource=resource, rule=resource.obj)
spec = resource.obj.get('spec', {})
# Validate the deltas
try:
deltas_str = resource.obj.get('spec', {}).get('deltas')
try:
deltas = parse_deltas(deltas_str)
except DeltasParseError as exc:
raise AnnotationError(
'Invalid delta string',
deltas_str=deltas_str
) from exc
if deltas is None or not deltas:
raise AnnotationError(
'parse_deltas returned invalid deltas',
deltas_str=deltas_str,
deltas=deltas,
)
except AnnotationError:
_log.exception(
'rule.invalid',
key_hints=['rule.metadata.name'],
)
return
# Refers to a disk from a cloud provider
if spec.get('disk'):
# Validate the backend
backend_name = spec.get('backend')
try:
backend = get_backend(backend_name)
except ConfigurationError as e:
_log.exception(
'rule.invalid',
message=e.message,
backend=backend_name
)
return
# Validate the disk identifier
disk = resource.obj.get('spec', {}).get('disk')
try:
disk = backend.validate_disk_identifier(disk)
except ValueError:
_log.exception(
'rule.invalid',
key_hints=['rule.metadata.name'],
)
return
rule = Rule(
name=rule_name_from_k8s_source(resource),
deltas=deltas,
backend=backend_name,
disk=disk
)
return rule
# Refers to a volume claim
if spec.get('persistentVolumeClaim'):
# Find the claim
volume_claim = await get_resource_or_none(
ctx.kube_client,
pykube.objects.PersistentVolumeClaim,
spec.get('persistentVolumeClaim'),
namespace=resource.metadata['namespace']
)
if not volume_claim:
_log.warning(
events.Rule.PENDING,
reason='Volume claim does not exist',
key_hints=['rule.metadata.name'],
)
raise RuleDependsOn(
'The volume claim targeted by this SnapshotRule does not exist yet',
kind='PersistentVolumeClaim',
namespace=resource.metadata['namespace'],
name=spec.get('persistentVolumeClaim')
)
# Find the volume
try:
volume = await volume_from_pvc(ctx, volume_claim)
except VolumeNotFound:
_log.warning(
events.Rule.PENDING,
reason='Volume claim is not bound',
key_hints=['rule.metadata.name'],
)
raise RuleDependsOn(
'The volume claim targeted by this SnapshotRule is not bound yet',
kind='PersistentVolumeClaim',
namespace=resource.metadata['namespace'],
name=spec.get('persistentVolumeClaim')
)
return await rule_from_pv(ctx, volume, deltas, source=resource)
async def rule_from_pv(
ctx: Context,
volume: pykube.objects.PersistentVolume,
deltas_annotation_key: str,
use_claim_name: bool = False,
) -> Rule:
"""Given a persistent volume object, create a backup role
object. Can return None if this volume is not configured for
backups, or is not suitable.
Parameters
`use_claim_name` - if the persistent volume is bound, and it's
name is auto-generated, then prefer to use the name of the claim
for the snapshot.
"""
_log = _logger.new(
volume=volume.obj,
annotation_key=deltas_annotation_key,
)
# Verify the provider
provisioner = volume.annotations.get('pv.kubernetes.io/provisioned-by')
_log = _log.bind(provisioner=provisioner)
if provisioner != 'kubernetes.io/gce-pd':
raise UnsupportedVolume('Unsupported provisioner',
provisioner=provisioner)
def get_deltas(annotations: Dict) -> List[timedelta]:
"""
Helper annotation-deltas-getter
Parameters
----------
annotations
Returns
-------
"""
try:
deltas_str = annotations[deltas_annotation_key]
except KeyError as exc:
raise AnnotationNotFound('No such annotation key',
key=deltas_annotation_key) from exc
if not deltas_str:
raise AnnotationError('Invalid delta string',
deltas_str=deltas_str)
try:
deltas = parse_deltas(deltas_str)
except DeltasParseError as exc:
raise AnnotationError('Invalid delta string',
deltas_str=deltas_str) from exc
if deltas is None or not deltas:
raise AnnotationError(
'parse_deltas returned invalid deltas',
deltas_str=deltas_str,
deltas=deltas,
)
return deltas
claim_ref = volume.obj['spec'].get('claimRef')
try:
_log.debug('Checking volume for deltas')
deltas = get_deltas(volume.annotations)
return Rule.from_volume(volume,
source=volume,
deltas=deltas,
use_claim_name=use_claim_name)
except AnnotationNotFound:
if claim_ref is None:
raise
volume_claim = await kube.get_resource_or_none(
ctx.kube_client,
pykube.objects.PersistentVolumeClaim,
claim_ref['name'],
namespace=claim_ref['namespace'],
)
if volume_claim is None:
raise AnnotationError(
'Could not find the PersistentVolumeClaim from claim_ref',
claim_ref=claim_ref,
)
try:
_log.debug('Checking volume claim for deltas')
deltas = get_deltas(volume_claim.annotations)
return Rule.from_volume(volume,
source=volume_claim,
deltas=deltas,
use_claim_name=use_claim_name)
except AnnotationNotFound as exc:
raise AnnotationNotFound('No deltas found via volume claim') from exc
async def rule_from_pv(
ctx: Context,
volume: pykube.objects.PersistentVolume,
deltas_annotation_key: str,
use_claim_name: bool = False,
) -> Rule:
"""Given a persistent volume object, create a backup rule
object. Can return None if this volume is not configured for
backups, or is not suitable.
The configuration for the rule will either come from the volume,
or it's claim, if one is associated.
`use_claim_name` - if the persistent volume is bound, and it's
name is auto-generated, then prefer to use the name of the claim
for the snapshot.
"""
_log = _logger.new(
volume=volume.obj,
annotation_key=deltas_annotation_key,
)
# Do we have a backend that supports this disk?
backend_name, backend_module = find_backend_for_volume(volume)
if not backend_module:
raise UnsupportedVolume('Unsupported volume', volume=volume)
else:
disk = backend_module.get_disk_identifier(volume)
_log.debug('Volume supported by backend',
volume=volume,
backend=backend_module,
disk=disk)
claim_ref = volume.obj['spec'].get('claimRef')
try:
_log.debug('Checking volume for deltas')
deltas = get_deltas(volume.annotations, deltas_annotation_key)
return Rule.from_volume(volume,
backend_name,
disk=disk,
source=volume,
deltas=deltas,
use_claim_name=use_claim_name)
except AnnotationNotFound:
if claim_ref is None:
raise
volume_claim = await kube.get_resource_or_none(
ctx.kube_client,
pykube.objects.PersistentVolumeClaim,
claim_ref['name'],
namespace=claim_ref['namespace'],
)
if volume_claim is None:
raise AnnotationError(
'Could not find the PersistentVolumeClaim from claim_ref',
claim_ref=claim_ref,
)
try:
_log.debug('Checking volume claim for deltas')
deltas = get_deltas(volume_claim.annotations, deltas_annotation_key)
return Rule.from_volume(volume,
backend_name,
disk=disk,
source=volume_claim,
deltas=deltas,
use_claim_name=use_claim_name)
except AnnotationNotFound as exc:
raise AnnotationNotFound('No deltas found via volume claim') from exc
async def rule_from_resource(
ctx: Context, resource: Union[pykube.objects.PersistentVolume,
pykube.objects.PersistentVolumeClaim,
SnapshotRule]
) -> Rule:
"""Given a Kubernetes resource, converts it to a snapshot `Rule`
instance, or returns None.
How this process works, depends on the resource given.
- If the resource is a volume, we read the disk and delta info
from there.
- If the resource is a volume claim, we look for the volume.
- A `SnapshotRule` custom resource directly defines the disk.
"""
_log = _logger.new(resource=resource, )
if (isinstance(resource, SnapshotRule)):
# Validate the backen
backend_name = resource.obj.get('spec', {}).get('backend')
try:
backend = get_backend(backend_name)
except ConfigurationError as e:
_log.exception('rule.invalid',
message=e.message,
backend=backend_name)
return
# Validate the deltas
try:
deltas_str = resource.obj.get('spec', {}).get('deltas')
try:
deltas = parse_deltas(deltas_str)
except DeltasParseError as exc:
raise AnnotationError('Invalid delta string',
deltas_str=deltas_str) from exc
if deltas is None or not deltas:
raise AnnotationError(
'parse_deltas returned invalid deltas',
deltas_str=deltas_str,
deltas=deltas,
)
except AnnotationError:
_log.exception(
'rule.invalid',
key_hints=['volume.metadata.name'],
)
return
# Validate the disk identifier
disk = resource.obj.get('spec', {}).get('disk')
try:
disk = backend.validate_disk_identifier(disk)
except ValueError:
_log.exception(
'rule.invalid',
key_hints=['volume.metadata.name'],
)
return
rule = Rule(name=rule_name_from_k8s_source(resource),
deltas=deltas,
backend=backend_name,
disk=disk)
return rule
if isinstance(resource, pykube.objects.PersistentVolumeClaim):
try:
volume = await volume_from_pvc(ctx, resource)
except VolumeNotFound:
_log.exception(
events.Volume.NOT_FOUND,
key_hints=[
'resource.metadata.name',
],
)
return
elif isinstance(resource, pykube.objects.PersistentVolume):
volume = resource
else:
raise RuntimeError(f'{resource} is not supported.')
volume_name = volume.name
_log = _log.bind(
volume_name=volume_name,
volume=volume.obj,
)
try:
rule = await rule_from_pv(
ctx,
volume,
ctx.config.get('deltas_annotation_key'),
use_claim_name=ctx.config.get('use_claim_name'))
return rule
except AnnotationNotFound as exc:
_log.info(
events.Annotation.NOT_FOUND,
key_hints=['volume.metadata.name'],
exc_info=exc,
)
except AnnotationError:
_log.exception(
events.Annotation.ERROR,
key_hints=['volume.metadata.name'],
)
except UnsupportedVolume as exc:
_log.info(
events.Volume.UNSUPPORTED,
key_hints=['volume.metadata.name'],
exc_info=exc,
)