def setUp(self):
self.config = {
"MONGO_DATABASE": "machine_restore_test",
"RPAAS_SERVICE_NAME": "test_rpaas_machine_restore",
"RESTORE_MACHINE_RUN_INTERVAL": 2,
"HOST_MANAGER": "fake"
}
self.storage = storage.MongoDBStorage(self.config)
colls = self.storage.db.collection_names(False)
for coll in colls:
self.storage.db.drop_collection(coll)
now = datetime.datetime.utcnow()
tasks = [
{"_id": "restore_10.1.1.1", "host": "10.1.1.1", "instance": "foo",
"created": now - datetime.timedelta(minutes=8)},
{"_id": "restore_10.2.2.2", "host": "10.2.2.2", "instance": "bar",
"created": now - datetime.timedelta(minutes=3)},
{"_id": "restore_10.3.3.3", "host": "10.3.3.3", "instance": "foo",
"created": now - datetime.timedelta(minutes=5)},
{"_id": "restore_10.4.4.4", "host": "10.4.4.4", "instance": "foo",
"created": now - datetime.timedelta(minutes=10)},
{"_id": "restore_10.5.5.5", "host": "10.5.5.5", "instance": "bar",
"created": now - datetime.timedelta(minutes=15)},
]
FakeManager.host_id = 0
FakeManager.hosts = ['10.1.1.1', '10.2.2.2', '10.3.3.3', '10.4.4.4', '10.5.5.5']
for task in tasks:
Host.create("fake", task['instance'], self.config)
self.storage.store_task(task)
redis.StrictRedis().delete("restore_machine:last_run")
def test_storage_use_uri_conf(self):
conf = {
"DBAAS_MONGODB_ENDPOINT": "mongodb://127.0.0.1:27017/some_other_db",
"MONGO_URI": "mongodb://127.0.0.1:27017/ignored",
}
storage.MongoDBStorage(conf)._hosts_collection().remove()
host_config = {
"HOST_ID": "fake-id-x"
}
host_config.update(conf)
h1 = Host.create('fake', 'my-group1', host_config)
stor = h1.storage()
self.assertEqual(stor.mongo_database, "some_other_db")
self.assertEqual(stor.db.name, "some_other_db")
h1.destroy()
db_host = Host.find('fake-id-x')
self.assertIsNone(db_host)
conf = {
"MONGO_URI": "mongodb://127.0.0.1:27017/now_used",
}
storage.MongoDBStorage(conf)._hosts_collection().remove()
host_config = {
"HOST_ID": "fake-id-x"
}
host_config.update(conf)
h1 = Host.create('fake', 'my-group1', host_config)
stor = h1.storage()
self.assertEqual(stor.mongo_database, "now_used")
self.assertEqual(stor.db.name, "now_used")
h1.destroy()
db_host = Host.find('fake-id-x')
self.assertIsNone(db_host)
def test_destroy_ignores_manager_error(self, log):
host = Host.create('fake', 'my-group', {"HOST_ID": "explode"})
self.assertEqual(host.id, "explode")
host.destroy()
self.assertEqual(log.call_args, call("Error trying to destroy host 'explode' "
"in 'fake': failure to destroy"))
db_host = Host.find('explode')
self.assertIsNone(db_host)
def test_restore_log_and_raises_exception_on_error(self, log):
host = Host.create('fake', 'my-group', {"HOST_ID": "explode"})
self.assertEqual(host.id, "explode")
self.assertRaises(Exception, host.restore)
self.assertEqual(log.call_args, call("Error trying to restore host 'explode' "
"in 'fake': failure to restore"))
db_host = Host.find('explode')
self.assertEqual(db_host.id, "explode")
def test_remove_host(self):
h1 = Host('x', 'x.me.com')
h2 = Host('y', 'y.me.com')
lb = LoadBalancer.create('fake', 'my-lb', {'LB_ID': 'explode'})
lb.add_host(h1)
lb.add_host(h2)
lb.remove_host(h1)
self.assertItemsEqual(lb.hosts, [h2])
db_lb = LoadBalancer.find('my-lb')
self.assertItemsEqual([h.to_json() for h in db_lb.hosts], [h2.to_json()])
def test_create_alternatives(self):
conf = {"HM_ALTERNATIVE_CONFIG_COUNT": "3"}
conf.update({"HOST_ID": "fake-1"})
host = Host.create('fake', 'my-group', conf)
self.assertEqual(host.alternative_id, 0)
conf.update({"HOST_ID": "fake-2"})
host = Host.create('fake', 'my-group', conf)
self.assertEqual(host.alternative_id, 1)
conf.update({"HOST_ID": "fake-3"})
host = Host.create('fake', 'my-group', conf)
self.assertEqual(host.alternative_id, 2)
conf.update({"HOST_ID": "fake-4"})
host = Host.create('fake', 'my-group', conf)
self.assertEqual(host.alternative_id, 0)
conf.update({"HOST_ID": "fake-5"})
host = Host.create('fake', 'my-group', conf)
self.assertEqual(host.alternative_id, 1)
conf.update({"HOST_ID": "fake-6"})
host = Host.create('fake', 'another-group', conf)
self.assertEqual(host.alternative_id, 0)
hosts = Host.list(filters={'alternative_id': 0})
self.assertEqual(len(hosts), 3)
hosts = Host.list(filters={'alternative_id': 1})
self.assertEqual(len(hosts), 2)
hosts = Host.list(filters={'alternative_id': 2})
self.assertEqual(len(hosts), 1)
def test_storage_use_database_conf(self):
storage.MongoDBStorage({"MONGO_DATABASE": "alternative_host_manager"})._hosts_collection().remove()
h1 = Host.create('fake', 'my-group1', {
"HOST_ID": "fake-id-x", "MONGO_DATABASE": "alternative_host_manager"
})
stor = h1.storage()
self.assertEqual(stor.mongo_database, "alternative_host_manager")
self.assertEqual(stor.db.name, "alternative_host_manager")
h1.destroy()
db_host = Host.find('fake-id-x')
self.assertIsNone(db_host)
def _restore_machine(self, task, config, healthcheck_timeout):
retry_failure_delay = int(self.config.get("RESTORE_MACHINE_FAILURE_DELAY", 5))
restore_dry_mode = self.config.get("RESTORE_MACHINE_DRY_MODE", False)
retry_failure_query = {"_id": {"$regex": "restore_.+"}, "last_attempt": {"$ne": None}}
if task['instance'] not in self._failure_instances(retry_failure_query, retry_failure_delay):
host = self.storage.find_host_id(task['host'])
if not restore_dry_mode:
Host.from_dict({"_id": host['_id'], "dns_name": task['host'],
"manager": host['manager']}, conf=config).restore()
self.nginx_manager.wait_healthcheck(task['host'], timeout=healthcheck_timeout)
self.storage.remove_task({"_id": task['_id']})
def test_list(self):
h1 = Host.create('fake', 'my-group1', {"HOST_ID": "fake-id-1"})
h2 = Host.create('fake', 'my-group1', {"HOST_ID": "fake-id-2"})
h3 = Host.create('fake', 'my-group2', {"HOST_ID": "fake-id-3"})
conf = {'MY_CONF': 1}
hosts = Host.list(conf=conf)
self.assertDictEqual(hosts[0].config, conf)
self.assertDictEqual(hosts[1].config, conf)
self.assertDictEqual(hosts[2].config, conf)
self.assertItemsEqual([h.to_json() for h in hosts], [h1.to_json(), h2.to_json(), h3.to_json()])
hosts = Host.list({'group': 'my-group1'})
self.assertItemsEqual([h.to_json() for h in hosts], [h1.to_json(), h2.to_json()])
def test_add_host(self):
h1 = Host('x', 'x.me.com')
h2 = Host('y', 'y.me.com')
conf = {'LB_ID': 'explode'}
lb = LoadBalancer.create('fake', 'my-lb', conf)
lb.add_host(h1)
lb.add_host(h2)
self.assertItemsEqual(lb.hosts, [h1, h2])
db_lb = LoadBalancer.find('my-lb', conf)
self.assertEqual(db_lb.hosts[0].config, conf)
self.assertEqual(db_lb.hosts[1].config, conf)
self.assertItemsEqual([h.to_json() for h in db_lb.hosts], [h1.to_json(), h2.to_json()])
def _add_host(self, lb):
host = Host.create(self.host_manager_name, lb.name, self.config)
try:
lb.add_host(host)
self.hc.add_url(lb.name, host.dns_name)
binding_data = self.storage.find_binding(lb.name)
if not binding_data:
return
self.nginx_manager.wait_healthcheck(host.dns_name, timeout=300)
cert, key = binding_data.get("cert"), binding_data.get("key")
if cert and key:
self.nginx_manager.update_certificate(host.dns_name, cert, key)
paths = binding_data.get("paths") or []
for path_data in paths:
self.nginx_manager.update_binding(
host.dns_name, path_data.get("path"), path_data.get("destination"), path_data.get("content")
)
except:
exc_info = sys.exc_info()
rollback = self._get_conf("RPAAS_ROLLBACK_ON_ERROR", "0") in ("True", "true", "1")
if not rollback:
raise
try:
host.destroy()
except Exception as e:
logging.error("Error in rollback trying to destroy host: {}".format(e))
try:
lb.remove_host(host)
except Exception as e:
logging.error("Error in rollback trying to remove from load balancer: {}".format(e))
try:
self.hc.remove_url(lb.name, host.dns_name)
except Exception as e:
logging.error("Error in rollback trying to remove healthcheck: {}".format(e))
raise exc_info[0], exc_info[1], exc_info[2]
def test_create(self):
conf = {"HOST_ID": "fake-id"}
host = Host.create('fake', 'my-group', conf)
self.assertEqual(host.id, "fake-id")
self.assertEqual(host.dns_name, "fake-id.my-group.com")
self.assertEqual(host.manager, "fake")
self.assertEqual(host.group, "my-group")
self.assertEqual(host.config, conf)
self.assertEqual(host.alternative_id, 0)
db_host = Host.find('fake-id', conf=conf)
self.assertEqual(db_host.id, "fake-id")
self.assertEqual(db_host.dns_name, "fake-id.my-group.com")
self.assertEqual(db_host.manager, "fake")
self.assertEqual(db_host.group, "my-group")
self.assertEqual(db_host.config, conf)
self.assertEqual(db_host.alternative_id, 0)
def run(self, config, name):
self.init_config(config)
healthcheck_timeout = int(self._get_conf("RPAAS_HEALTHCHECK_TIMEOUT", 600))
host = Host.create(self.host_manager_name, name, self.config)
lb = None
try:
lb = LoadBalancer.create(self.lb_manager_name, name, self.config)
lb.add_host(host)
self.nginx_manager.wait_healthcheck(host.dns_name, timeout=healthcheck_timeout)
self.hc.create(name)
self.hc.add_url(name, host.dns_name)
self.storage.remove_task(name)
except:
exc_info = sys.exc_info()
rollback = self._get_conf("RPAAS_ROLLBACK_ON_ERROR", "0") in ("True", "true", "1")
if not rollback:
raise
try:
if lb is not None:
lb.destroy()
except Exception as e:
logging.error("Error in rollback trying to destroy load balancer: {}".format(e))
try:
host.destroy()
except Exception as e:
logging.error("Error in rollback trying to destroy host: {}".format(e))
try:
self.hc.destroy(name)
except Exception as e:
logging.error("Error in rollback trying to remove healthcheck: {}".format(e))
raise exc_info[0], exc_info[1], exc_info[2]
def _restore_machine(self, task, config, healthcheck_timeout):
retry_failure_delay = int(self.config.get("RESTORE_MACHINE_FAILURE_DELAY", 5))
restore_dry_mode = self.config.get("RESTORE_MACHINE_DRY_MODE", False) in ("True", "true", "1")
retry_failure_query = {"_id": {"$regex": "restore_.+"}, "last_attempt": {"$ne": None}}
if task['instance'] not in self._failure_instances(retry_failure_query, retry_failure_delay):
host = self.storage.find_host_id(task['host'])
if not restore_dry_mode:
healing_id = self.storage.store_healing(task['instance'], task['host'])
try:
Host.from_dict({"_id": host['_id'], "dns_name": task['host'],
"manager": host['manager']}, conf=config).restore()
Host.from_dict({"_id": host['_id'], "dns_name": task['host'],
"manager": host['manager']}, conf=config).start()
self.nginx_manager.wait_healthcheck(task['host'], timeout=healthcheck_timeout)
self.storage.update_healing(healing_id, "success")
except Exception as e:
self.storage.update_healing(healing_id, str(e.message))
raise e
self.storage.remove_task({"_id": task['_id']})
def from_dict(cls, dict, conf=None):
if dict is None:
return None
dict['name'] = dict['_id']
del dict['_id']
dict['conf'] = conf
hosts_data = dict.get('hosts', None)
if hosts_data:
dict['hosts'] = [Host.from_dict(h, conf=conf) for h in hosts_data]
return cls(**dict)
def run(self, config):
self.init_config(config)
retry_failure_delay = int(self.config.get("RESTORE_MACHINE_FAILURE_DELAY", 5))
restore_dry_mode = self.config.get("RESTORE_MACHINE_DRY_MODE", False)
retry_failure_query = {"_id": {"$regex": "restore_.+"}, "last_attempt": {"$ne": None}}
failure_instances = set()
for task in self.storage.find_task(retry_failure_query):
retry_failure = task['last_attempt'] + datetime.timedelta(minutes=retry_failure_delay)
if (retry_failure >= datetime.datetime.utcnow()):
failure_instances.add(task['instance'])
restore_delay = int(self.config.get("RESTORE_MACHINE_DELAY", 5))
created_in = datetime.datetime.utcnow() - datetime.timedelta(minutes=restore_delay)
query = {"_id": {"$regex": "restore_.+"}, "created": {"$lte": created_in}}
for task in self.storage.find_task(query):
try:
if task['instance'] not in failure_instances:
host = self.storage.find_host_id(task['host'])
if not restore_dry_mode:
Host.from_dict({"_id": host['_id'], "dns_name": task['host'],
"manager": host['manager']}, conf=config).restore()
self.storage.remove_task({"_id": task['_id']})
except Exception as e:
self.storage.update_task(task['_id'], {"last_attempt": datetime.datetime.utcnow()})
raise e
def _add_host(self, name, lb=None):
healthcheck_timeout = int(self._get_conf("RPAAS_HEALTHCHECK_TIMEOUT", 600))
created_lb = None
try:
if not lb:
lb = created_lb = LoadBalancer.create(self.lb_manager_name, name, self.config)
self.hc.create(name)
config = copy.deepcopy(self.config)
if hasattr(lb, 'dsr') and lb.dsr:
config["HOST_TAGS"] = config["HOST_TAGS"] + ",dsr_ip:{}".format(lb.address)
host = Host.create(self.host_manager_name, name, config)
lb.add_host(host)
self.nginx_manager.wait_healthcheck(host.dns_name, timeout=healthcheck_timeout)
acls = self.consul_manager.find_acl_network(name)
if acls:
acl_host = acls.pop()
for dst in acl_host['destination']:
self.acl_manager.add_acl(name, host.dns_name, dst)
self.hc.add_url(name, host.dns_name)
except:
exc_info = sys.exc_info()
rollback = self._get_conf("RPAAS_ROLLBACK_ON_ERROR", "0") in ("True", "true", "1")
if not rollback:
raise
try:
if created_lb is not None:
created_lb.destroy()
except Exception as e:
logging.error("Error in rollback trying to destroy load balancer: {}".format(e))
try:
if created_lb is not None:
self._delete_host(name, host)
else:
self._delete_host(name, host, lb)
except Exception as e:
logging.error("Error in rollback trying to destroy host: {}".format(e))
try:
if lb and len(lb.hosts) == 0:
self.hc.destroy(name)
except Exception as e:
logging.error("Error in rollback trying to remove healthcheck: {}".format(e))
raise exc_info[0], exc_info[1], exc_info[2]
finally:
self.storage.remove_task(name)
def test_tag_vm(self):
host = Host.create('fake', 'my-group', {"HOST_ID": "fake-id"})
self.assertEqual(host.id, "fake-id")
tags = ['a:b', 'c:d']
host.tag_vm(tags)
def create_host(self, name=None, alternative_id=0):
id = self.host_id
FakeManager.host_id += 1
return Host(id=id,
dns_name=FakeManager.hosts.pop(0),
alternative_id=alternative_id)
def test_restore(self):
host = Host.create('fake', 'my-group', {"HOST_ID": "fake-id"})
self.assertEqual(host.id, "fake-id")
host.restore()
def test_destroy_ignores_manager_error(self):
host = Host.create('fake', 'my-group', {"HOST_ID": "explode"})
self.assertEqual(host.id, "explode")
host.destroy()
db_host = Host.find('explode')
self.assertIsNone(db_host)
def test_destroy(self):
host = Host.create('fake', 'my-group', {"HOST_ID": "fake-id"})
self.assertEqual(host.id, "fake-id")
host.destroy()
db_host = Host.find('fake-id')
self.assertIsNone(db_host)
def test_restore(self):
host = Host.create('fake', 'my-group', {"HOST_ID": "fake-id"})
self.assertEqual(host.id, "fake-id")
host.restore()