def getAndcheckAllParameters():
configService = Config()
settings = configService.getSettings()
try:
checkParameters(settings)
except KeyError as e:
raise Exception("Somthing wrong on your config file: %s" % e.message)
logger.info("Rancher URL: %s",
settings['rancher']['api']['url'][:-2] + "v2-beta")
logger.info("Rancher key: %s", settings['rancher']['api']['key'])
logger.info("Rancher secret: XXXX")
logger.info("Backup path: %s", settings['duplicity']['source-path'])
logger.info("Backup target path: %s", settings['duplicity']['target-path'])
logger.info("Backend to receive remote backup: %s",
settings['duplicity']['url'])
logger.info("Backup full frequency: %s",
settings['duplicity']['full-if-older-than'])
logger.info("Backup full to keep: %s",
settings['duplicity']['remove-all-but-n-full'])
logger.info("Backup incremental chain to keep: %s",
settings['duplicity']['remove-all-inc-of-but-n-full'])
logger.info("Volume size: %s", settings['duplicity']['volsize'])
logger.info("Backup options: %s", settings['duplicity']['options'])
# Init services
try:
rancherService = Rancher(
settings['rancher']['api']['url'][:-2] + "v2-beta",
settings['rancher']['api']['key'],
settings['rancher']['api']['secret'])
except Exception as e:
raise Exception("Can't connect to rancher API : %s \n%s" %
(e.message, traceback.format_exc()))
try:
rancherDatabaseSettings = rancherService.getDatabaseSettings()
except Exception as e:
rancherDatabaseSettings = {}
pass
# Check database settings
try:
rancherDatabaseSettings = checkAndGetDatabaseSettings(
settings, rancherDatabaseSettings)
except KeyError as e:
raise Exception(
"You must set the Rancher database settings on config file to dump it: %s"
% e.message)
return (settings, rancherDatabaseSettings)
def testGetTemplate(self, run_mock):
Config._drop()
configService = Config("/fake/path")
template = configService.getTemplate("mysql.yml")
targetTemplate = "my mysql fake sample"
self.assertEqual(template, targetTemplate)
configService = Config()
template = configService.getTemplate("mysql.yml")
self.assertEqual(template, targetTemplate)
def testGetSettings(self, run_mock):
Config._drop()
configService = Config("/fake/path")
settings = configService.getSettings()
targetSettings = {'rancher.host': "test", 'rancher.port': 1234}
self.assertEqual(settings, targetSettings)
configService = Config()
settings = configService.getSettings()
self.assertEqual(settings, targetSettings)
def testGetIndex(self, run_mock):
Config._drop()
configService = Config("/fake/path")
index = configService.getIndex()
targetIndex = {
'mysql': {
'image': 'fake/image',
'command': 'facke command'
}
}
self.assertEqual(index, targetIndex)
configService = Config()
index = configService.getIndex()
self.assertEqual(index, targetIndex)
def testGetConfig(self, run_mock):
configService = Config("/fake/path")
settings = configService.getConfig()
targetSettings = {
'mysql': {
'image': 'fake/image',
'command': 'facke command'
}
}
self.assertEqual(settings, targetSettings)
configService = Config()
settings = configService.getConfig()
self.assertEqual(settings, targetSettings)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
console_handler = logging.StreamHandler()
console_handler.setLevel(loglevel)
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)
file_handler = TimedRotatingFileHandler('/var/log/backup/backup.log',
when='d',
interval=1,
backupCount=5)
file_handler.setLevel(loglevel)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
# Load and check settings
configService = Config("config")
# Just check parameters
if len(sys.argv) > 1 and sys.argv[1] == "--checkParameters":
try:
getAndcheckAllParameters()
except Exception as e:
logger.error("Error - %s", e.message)
sys.exit(1)
# Run backup
else:
try:
(settings, rancherDatabaseSettings) = getAndcheckAllParameters()
logger.info("Backup full to keep: %s", BK_KEEP_FULL)
logger.info("Backup incremental chain to keep: %s", BK_KEEP_FULL_CHAIN)
logger.info("Volume size: %s", VOLUME_SIZE)
# Init services
try:
rancherService = Rancher(os.getenv("CATTLE_URL"),
os.getenv("CATTLE_ACCESS_KEY"),
os.getenv("CATTLE_SECRET_KEY"))
except Exception as e:
logger.error("Can't connect to rancher API : %s", e.message)
logger.error(traceback.format_exc())
sys.exit(1)
try:
configService = Config("config/*.yml")
except Exception as e:
logger.error("Can't load settings or syntax errors : %s", e.message)
logger.error(traceback.format_exc())
sys.exit(1)
backupService = Backup()
backend = os.getenv('BACKEND') + TARGET_PATH
try:
# Load settings
listSettings = configService.getConfig()
# We init duplicity
try:
def testSearchDump(self, mock_config):
backupService = Backup()
Config('/fake/path')
# Search template backup with image name
listServices = [{
'type':
'service',
'name':
'test',
'state':
'active',
'launchConfig': {
'imageUuid': 'test/postgres:latest',
'environment': {
'POSTGRES_USER': '******',
'POSTGRES_DB': 'test',
'POSTGRES_PASSWORD': '******'
}
},
'links': {
'environment': 'https://fake/environment',
'instances': 'https://fake/instances',
},
'stack': {
'name': 'stack-test'
},
'instances': [{
'state': 'disabled',
'primaryIpAddress': '10.0.0.1',
'host': {
'name': 'host-1'
},
'links': {
'hosts': 'https://fake/hosts'
}
}, {
'state': 'running',
'primaryIpAddress': '10.0.0.2',
'host': {
'name': 'host-1'
},
'links': {
'hosts': 'https://fake/hosts'
}
}, {
'state': 'running',
'primaryIpAddress': '10.0.0.3',
'host': {
'name': 'host-1'
},
'links': {
'hosts': 'https://fake/hosts'
}
}],
}, {
'type':
'service',
'name':
'test2',
'state':
'active',
'launchConfig': {
'imageUuid': 'test/mysql:latest',
'environment': {
'MYSQL_USER': '******',
'MYSQL_DATABASE': 'test',
'MYSQL_PASSWORD': '******'
}
},
'links': {
'environment': 'https://fake/environment',
'instances': 'https://fake/instances',
},
'stack': {
'name': 'stack-test'
},
'instances': [{
'state': 'disabled',
'primaryIpAddress': '10.1.0.1',
'host': {
'name': 'host-1'
},
'links': {
'hosts': 'https://fake/hosts'
}
}, {
'state': 'running',
'primaryIpAddress': '10.1.0.2',
'host': {
'name': 'host-1'
},
'links': {
'hosts': 'https://fake/hosts'
}
}, {
'state': 'running',
'primaryIpAddress': '10.1.0.3',
'host': {
'name': 'host-1'
},
'links': {
'hosts': 'https://fake/hosts'
}
}],
}]
result = backupService.searchDump('/tmp/backup', listServices)
targetResult = [{
'service':
listServices[0],
'target_dir':
'/tmp/backup/stack-test/test',
'commands': [
'pg_dump -h 10.0.0.2 -U user -d test -f /tmp/backup/stack-test/test/test.dump'
],
'environments': ['PGPASSWORD:pass'],
'image':
'postgres:latest'
}]
self.assertEqual(targetResult, result)
# Search backup with labels
# Search template backup with image name
listServices = [{
'type':
'service',
'name':
'test',
'state':
'active',
'launchConfig': {
'imageUuid': 'test/my-db:latest',
'environment': {
'POSTGRES_USER': '******',
'POSTGRES_DB': 'test',
'POSTGRES_PASSWORD': '******'
},
'labels': {
'backup.type': 'postgres'
}
},
'links': {
'environment': 'https://fake/environment',
'instances': 'https://fake/instances',
},
'stack': {
'name': 'stack-test'
},
'instances': [{
'state': 'disabled',
'primaryIpAddress': '10.0.0.1',
'host': {
'name': 'host-1'
},
'links': {
'hosts': 'https://fake/hosts'
}
}, {
'state': 'running',
'primaryIpAddress': '10.0.0.2',
'host': {
'name': 'host-1'
},
'links': {
'hosts': 'https://fake/hosts'
}
}, {
'state': 'running',
'primaryIpAddress': '10.0.0.3',
'host': {
'name': 'host-1'
},
'links': {
'hosts': 'https://fake/hosts'
}
}],
}, {
'type':
'service',
'name':
'test2',
'state':
'active',
'launchConfig': {
'imageUuid': 'test/my-db:latest',
'environment': {
'MYSQL_USER': '******',
'MYSQL_DATABASE': 'test',
'MYSQL_PASSWORD': '******'
},
'labels': {
'backup.type': 'mysql'
}
},
'links': {
'environment': 'https://fake/environment',
'instances': 'https://fake/instances',
},
'stack': {
'name': 'stack-test'
},
'instances': [{
'state': 'disabled',
'primaryIpAddress': '10.1.0.1',
'host': {
'name': 'host-1'
},
'links': {
'hosts': 'https://fake/hosts'
}
}, {
'state': 'running',
'primaryIpAddress': '10.1.0.2',
'host': {
'name': 'host-1'
},
'links': {
'hosts': 'https://fake/hosts'
}
}, {
'state': 'running',
'primaryIpAddress': '10.1.0.3',
'host': {
'name': 'host-1'
},
'links': {
'hosts': 'https://fake/hosts'
}
}],
}]
result = backupService.searchDump('/tmp/backup', listServices)
targetResult = [{
'service':
listServices[0],
'target_dir':
'/tmp/backup/stack-test/test',
'commands': [
'pg_dump -h 10.0.0.2 -U user -d test -f /tmp/backup/stack-test/test/test.dump'
],
'environments': ['PGPASSWORD:pass'],
'image':
'postgres:latest'
}]
self.assertEqual(targetResult, result)
def searchDump(self, backupPath, listServices):
"""
This class search service where to perform a dump before to backup them and grab all setting to perform backup
:param backupPath: The path where to store the dump
:param listServices: The list of all services provider by Rancher API
:type backupPath: str
:type listServices: list
:return dict The list of service where to perform dump and docker command line associated to them.
"""
if backupPath is None or backupPath == "":
raise KeyError("backupPath must be provided")
if isinstance(listServices, list) is False:
raise KeyError("listServices must be a list")
logger.debug("backupPath: %s", backupPath)
logger.debug("listServices: %s", listServices)
configService = Config()
index = configService.getIndex()
listDump = []
for service in listServices:
for name, setting in index.iteritems():
if re.search(setting['regex'],
service['launchConfig']['imageUuid']):
logger.info("Found '%s/%s' to do dumping" %
(service['stack']['name'], service['name']))
template = configService.getTemplate(setting['template'])
env = Environment()
template = env.from_string(template)
context = {}
# Get environment variables
if 'environment' in service['launchConfig']:
context["env"] = service['launchConfig']['environment']
# Get IP
for instance in service['instances']:
if instance['state'] == "running":
context["ip"] = instance['primaryIpAddress']
logger.debug("Found IP %s", context["ip"])
break
# Get Taget backup
context["target_dir"] = backupPath + "/" + service[
'stack']['name'] + "/" + service['name']
setting = yaml.load(template.render(context))
setting["service"] = service
setting["target_dir"] = context["target_dir"]
if "environments" not in setting:
setting["environments"] = []
if "image" not in setting:
setting["image"] = service['launchConfig']['imageUuid']
listDump.append(setting)
break
logger.debug(listDump)
return listDump
def tearDown(self):
Config._drop()
def setUp(self):
configService = Config("../../../../config")
def setUp(self):
self.maxDiff = None
configService = Config("../../config")