def main(argv):
if "-h" in argv:
print "./simulate.py [backup-timestamps]"
return 1
# Do some default simulation
if not argv:
s = BackupSimulator(parse_deltas("1d 7d 30d"))
until = s.now + timedelta(days=17)
while s.now <= until:
s.go_by(timedelta(days=1))
s.backup()
for name, date in s.backups.iteritems():
print name
# Simulate a backup with the timestamps given
else:
s = BackupSimulator(parse_deltas("1d 7d 30d"))
s.add([d for d in argv])
deleted, _ = s.expire()
print "Deleted backups:"
for name in deleted:
print (name)
print ""
print "Remaining backups:"
for name, date in s.backups.iteritems():
print name
def main(argv):
if '-h' in argv:
print "./simulate.py [backup-timestamps]"
return 1
# Do some default simulation
if not argv:
s = BackupSimulator(parse_deltas('1d 7d 30d'))
until = s.now + timedelta(days=17)
while s.now <= until:
s.go_by(timedelta(days=1))
s.backup()
for name, date in s.backups.iteritems():
print name
# Simulate a backup with the timestamps given
else:
s = BackupSimulator(parse_deltas('1d 7d 30d'))
s.add([d for d in argv])
deleted, _ = s.expire()
print "Deleted backups:"
for name in deleted:
print(name)
print ""
print "Remaining backups:"
for name, date in s.backups.iteritems():
print name
def __init__(self, deltas, expire_func=default_expire_func):
if isinstance(deltas, basestring):
deltas = parse_deltas(deltas)
self.deltas = deltas
self.expire_func = expire_func
self.now = datetime.now()
self.backups = OrderedDict()
def read_volume(name):
_log = _logger.new(volume_name=name, )
env_name = name.replace('-', '_').upper()
deltas_str = os.environ.get('VOLUME_{}_DELTAS'.format(env_name))
if not deltas_str:
raise ConfigError(
'A volume {} was defined, but VOLUME_{}_DELTAS is not set'.
format(name, env_name))
zone = os.environ.get('VOLUME_{}_ZONE'.format(env_name))
if not zone:
raise ConfigError(
'A volume {} was defined, but VOLUME_{}_ZONE is not set'.
format(name, env_name))
_log = _log.bind(
deltas_str=deltas_str,
zone=zone,
)
rule = Rule(
name=name,
claim_name='',
namespace='',
deltas=parse_deltas(deltas_str),
gce_disk=name,
gce_disk_zone=zone,
)
_log.info(events.Rule.ADDED_FROM_CONFIG, rule=rule)
return rule
def rule_from_pv(volume, use_claim_name=False):
"""Given a persistent volume object, create a backup role
object. Can return None if this volume is not configured for
backups, or is not suitable.
`use_claim_name` - if the persistent volume is bound, and it's
name is auto-generated, then prefer to use the name of the claim
for the snapshot.
"""
# TODO: Currently, K8s does not allow a PersistentVolumeClaim to
# specify any annotations for the PersistentVolume a provisioner
# would create. Indeed, this might ever be possible. We might
# want to follow the claimRef link and see if the claim specifies
# any rules, and then use those.
provider = volume.annotations.get('pv.kubernetes.io/provisioned-by')
if provider != 'kubernetes.io/gce-pd':
logger.debug('Volume {} not a GCE persistent disk', volume.name)
return
deltas_unparsed = volume.annotations.get('backup.kubernetes.io/deltas')
if not deltas_unparsed:
logger.debug('Volume {} does not define backup deltas (via {})',
volume.name, DELTA_ANNOTATION_KEY)
return
try:
deltas = parse_deltas(deltas_unparsed)
except ConfigError as e:
logger.error(
'Deltas defined by volume {} are not valid, error message was: {}',
volume.name, e)
return
rule = Rule()
rule.name = volume.name
rule.namespace = volume.namespace
rule.deltas = deltas
rule.deltas_unparsed = deltas_unparsed
rule.gce_disk = volume.obj['spec']['gcePersistentDisk']['pdName']
# How can we know the zone? In theory, the storage class can
# specify a zone; but if not specified there, K8s can choose a
# random zone within the master region. So we really can't trust
# that value anyway.
# There is a label that gives a failure region, but labels aren't
# really a trustworthy source for this.
# Apparently, this is a thing in the Kubernetes source too, see:
# getDiskByNameUnknownZone in pkg/cloudprovider/providers/gce/gce.go,
# e.g. https://github.com/jsafrane/kubernetes/blob/2e26019629b5974b9a311a9f07b7eac8c1396875/pkg/cloudprovider/providers/gce/gce.go#L2455
rule.gce_disk_zone = volume.labels.get(
'failure-domain.beta.kubernetes.io/zone')
if use_claim_name and volume.obj['spec'].get('claimRef'):
if volume.annotations.get(
'kubernetes.io/createdby') == 'gce-pd-dynamic-provisioner':
ref = volume.obj['spec'].get('claimRef')
rule.claim_name = '{1}--{0}'.format(ref['name'], ref['namespace'])
return rule
def job(self, deltas='1d 2d', name='test', **kwargs):
"""Make a job object.
"""
opts = dict(target="$name-$date",
deltas=parse_deltas(deltas),
name=name,
sources=[self._tmpdir])
opts.update(kwargs)
return Job(**opts)
def job(self, deltas='1d 2d', name='test', **kwargs):
"""Make a job object.
"""
opts = dict(
target="$name-$date",
deltas=parse_deltas(deltas),
name=name,
sources=[self._tmpdir])
opts.update(kwargs)
return Job(**opts)
def main(argv):
s = BackupSimulator(parse_deltas('1d 7d 30d'))
until = s.now + timedelta(days=17)
while s.now <= until:
s.go_by(timedelta(days=1))
s.backup()
for name, date in s.backups.iteritems():
print name
def read_volume(name):
env_name = name.replace('-', '_').upper()
deltas = os.environ.get('VOLUME_{}_DELTAS'.format(env_name))
if not deltas:
raise ConfigError(
'A volume {} was defined, but no deltas'.format(name))
zone = os.environ.get('VOLUME_{}_ZONE'.format(env_name))
if not zone:
raise ConfigError(
'A volume {} was defined, but no zone'.format(name))
logger.info('Loading env-defined volume {} with deltas {}', name,
deltas)
rule = Rule()
rule.name = name
rule.namespace = ''
rule.deltas = parse_deltas(deltas)
rule.deltas_unparsed = deltas
rule.gce_disk = name
rule.gce_disk_zone = zone
return rule
def load_config(text):
"""Load the config file and return a dict of jobs, with the local
and global configurations merged.
"""
config = yaml.load(text)
default_dateformat = config.pop('dateformat', None)
default_deltas = parse_deltas(config.pop('deltas', None))
default_target = require_placeholders(config.pop('target', None),
['name', 'date'], 'The global target')
read_jobs = {}
jobs_section = config.pop('jobs')
if not jobs_section:
raise ConfigError('config must define at least one job')
for job_name, job_dict in jobs_section.iteritems():
job_dict = job_dict or {}
# sources
if 'sources' in job_dict and 'source' in job_dict:
raise ConfigError(('%s: Use either the "source" or "sources" '+
'option, not both') % job_name)
if 'source' in job_dict:
sources = [job_dict.pop('source')]
else:
sources = job_dict.pop('sources', None)
# aliases
if 'aliases' in job_dict and 'alias' in job_dict:
raise ConfigError(('%s: Use either the "alias" or "aliases" '+
'option, not both') % job_name)
if 'alias' in job_dict:
aliases = [job_dict.pop('alias')]
else:
aliases = job_dict.pop('aliases', None)
# excludes
if 'excludes' in job_dict and 'exclude' in job_dict:
raise ConfigError(('%s: Use either the "excludes" or "exclude" '+
'option, not both') % job_name)
if 'exclude' in job_dict:
excludes = [job_dict.pop('exclude')]
else:
excludes = job_dict.pop('excludes', [])
new_job = Job(**{
'name': job_name,
'sources': sources,
'aliases': aliases,
'excludes': excludes,
'target': job_dict.pop('target', default_target),
'force': job_dict.pop('force', False),
'deltas': parse_deltas(job_dict.pop('deltas', None)) or default_deltas,
'dateformat': job_dict.pop('dateformat', default_dateformat),
'exec_before': job_dict.pop('exec_before', None),
'exec_after': job_dict.pop('exec_after', None),
})
if not new_job.target:
raise ConfigError('%s does not have a target name' % job_name)
# Note: It's ok to define jobs without sources or deltas. Those
# can only be used for selected commands, then.
require_placeholders(new_job.target, ['date'], '%s: target')
if job_dict:
raise ConfigError('%s has unsupported configuration values: %s' % (
job_name, ", ".join(job_dict.keys())))
read_jobs[job_name] = new_job
# Return jobs, and all global keys not popped
return read_jobs, config
