def _choose_active_instances(services_dicts):
result = services_dicts
running_services_with_single_active_instances = {}
for microservice_id, service_dict in services_dicts.items():
if service_dict.get('single_active_instance'
) and service_dict['status'] in ('passing',
'warning'):
key = 'chosen_active_instance/{},env={},app_id={}'.format(
service_dict['name'], service_dict['tags'].get('env') or '',
service_dict['tags'].get('app_id') or '')
if key not in running_services_with_single_active_instances:
running_services_with_single_active_instances[key] = set()
running_services_with_single_active_instances[key].add(
microservice_id)
for key, running_microservice_ids in running_services_with_single_active_instances.items(
):
currently_picked_instance = kv.kv_get(key)
if currently_picked_instance not in running_microservice_ids:
currently_picked_instance = random.choice(
list(running_microservice_ids))
kv.kv_set(key, currently_picked_instance)
for microservice_id in running_microservice_ids:
if microservice_id != currently_picked_instance:
result[microservice_id]['status'] = 'standby'
return result
def register_service_in_consul(microservice_data):
if _exists_service(microservice_data['microservice_id']):
return
consul_service_data = {
'ID': microservice_data['microservice_id'],
'Name': microservice_data['microservice_name'],
'Port': microservice_data['microservice_port'],
'Check': {
'TTL': '15s',
}
}
if microservice_data['microservice_tags']:
consul_service_data['Tags'] = microservice_data['microservice_tags']
response = consul_post('agent/service/register', consul_service_data)
response.raise_for_status()
container_id = microservice_data['microservice_id'].split(':')[0]
kv_set('start_timestamp/{}'.format(container_id), str(microservice_data['container_created_timestamp']))
key = 'single_active_instance/{}'.format(microservice_data['microservice_id'])
if microservice_data.get('single_active_instance'):
kv_set(key, True)
else:
kv_remove(key)
def save_container(ship, container_id, status, params=None):
try:
start_timestamp = kv_get('start_timestamp/{}'.format(container_id))
except:
start_timestamp = None
if status == 'crashed':
service_name = params['microservice_name']
else:
service_name = get_env(container_id, 'MICROSERVICE_NAME')
params = json.loads(
base64.b64decode(
get_env(container_id, 'RESTART_CONTAINER_PARAMETERS')))
if not start_timestamp:
start_timestamp = str(calendar.timegm(time.gmtime()))
address = kv_get('ships/{}/ip'.format(ship)) or ship
service_dict = {
'ServiceName': service_name,
'Status': status,
'container_id': container_id,
'params': params,
'start_timestamp': start_timestamp,
'ServiceID': container_id,
'Address': address
}
kv_set(create_consul_services_key(ship, service_name, container_id),
service_dict)
def register_service_in_consul(microservice_data):
if exists_service(microservice_data['microservice_id']):
return
consul_service_data = {
'ID': microservice_data['microservice_id'],
'Name': microservice_data['microservice_name'],
'Port': microservice_data['microservice_port'],
'Check': {
'TTL': '15s',
}
}
microservice_tags = microservice_data.get('microservice_tags')
if microservice_tags:
consul_service_data['Tags'] = microservice_tags
response = consul_post('agent/service/register', consul_service_data)
response.raise_for_status()
container_id = microservice_data['microservice_id'].split(':')[0]
kv_set('start_timestamp/{}'.format(container_id),
str(microservice_data['container_created_timestamp']))
key = 'single_active_instance/{}'.format(
microservice_data['microservice_id'])
if microservice_data.get('single_active_instance'):
kv_set(key, True)
else:
kv_remove(key)
def update_container_status(status, ship=None, service_name=None, container_id=None, key=None):
if not key:
key = create_consul_services_key(ship, service_name, container_id)
service_dict = kv_get(key)
if status == 'crashed' and service_dict['Status'] in ['not-recovered', 'recovering']:
return
service_dict['Status'] = status
kv_set(key, service_dict)
def _load_from_dict(saved_containers, containers_saved_in_kv, ship):
for key, container_dict in saved_containers.items():
old_ship_name = key.split('/')[1]
if old_ship_name != ship:
key = 'ships/{}/service/{}/{}'.format(ship, container_dict['ServiceName'],
container_dict['container_id'])
if not containers_saved_in_kv or key not in containers_saved_in_kv:
kv.kv_set(key, container_dict)
def set_alias(name, address, user=None, password=None):
key = 'dockyard/aliases/{name}'.format(**locals())
value = {'address': address}
if user:
value['user'] = user
if password:
value['password'] = password
will_be_default = len(get_list()) <= 1 and name != DOCKYARD_FALLBACK_ALIAS
kv.kv_set(key, value)
if will_be_default:
set_default(name)
def set_alias(name, address, user=None, password=None):
key = 'dockyard/aliases/{name}'.format(**locals())
value = {'address': address}
if user:
value['user'] = user
if password:
value['password'] = password
will_be_default = len(get_list()) <= 1 and name != DOCKYARD_FALLBACK_ALIAS
kv.kv_set(key, value)
if will_be_default:
set_default(name)
def on_post(self, req, resp):
consul_host, error = self.get_post_parameter(req, 'host')
if error:
return self.status_error(resp, error)
ship = get_ship_name()
local_services_data = {
key: kv.kv_get(key)
for key in get_local_services_from_kv_store()
}
armada_size = _get_armada_size()
if armada_size > 1:
return self.status_error(
resp,
'Currently only single ship armadas can join the others. '
'Your armada has size: {0}.'.format(armada_size))
try:
agent_self_dict = consul_query(
'agent/self', consul_address='{0}:8500'.format(consul_host))
datacenter = agent_self_dict['Config']['Datacenter']
except Exception as e:
get_logger().exception(e)
return self.status_error(
resp, 'Could not read remote host datacenter address.')
current_consul_mode = _get_current_consul_mode()
if current_consul_mode == consul_config.ConsulMode.BOOTSTRAP:
override_runtime_settings(
consul_mode=consul_config.ConsulMode.CLIENT,
ship_ips=[consul_host],
datacenter=datacenter)
else:
override_runtime_settings(ship_ips=[consul_host] +
get_other_ship_ips(),
datacenter=datacenter)
if _restart_consul():
supervisor_server = xmlrpc.client.Server(
'http://localhost:9001/RPC2')
hermes_init_output = supervisor_server.supervisor.startProcessGroup(
'hermes_init')
get_logger().info('hermes_init start: %s', hermes_init_output)
set_ship_name(ship)
for key, data in six.iteritems(local_services_data):
kv.kv_set(key, data)
return self.status_ok(resp)
return self.status_error(resp, 'Waiting for armada restart timed out.')
def _choose_active_instances(services_dicts):
result = services_dicts
running_services_with_single_active_instances = {}
for microservice_id, service_dict in services_dicts.items():
if service_dict.get('single_active_instance') and service_dict['status'] in ('passing', 'warning'):
key = 'chosen_active_instance/{},env={},app_id={}'.format(service_dict['name'],
service_dict['tags'].get('env') or '',
service_dict['tags'].get('app_id') or '')
if key not in running_services_with_single_active_instances:
running_services_with_single_active_instances[key] = set()
running_services_with_single_active_instances[key].add(microservice_id)
for key, running_microservice_ids in running_services_with_single_active_instances.items():
currently_picked_instance = kv.kv_get(key)
if currently_picked_instance not in running_microservice_ids:
currently_picked_instance = random.choice(list(running_microservice_ids))
kv.kv_set(key, currently_picked_instance)
for microservice_id in running_microservice_ids:
if microservice_id != currently_picked_instance:
result[microservice_id]['status'] = 'standby'
return result
def POST(self):
consul_host, error = self.get_post_parameter('host')
if error:
return self.status_error(error)
ship = get_ship_name()
local_services = kv.kv_list('ships/{}/service/'.format(ship)) or []
local_services_data = {key: kv.kv_get(key) for key in local_services}
armada_size = _get_armada_size()
if armada_size > 1:
return self.status_error('Currently only single ship armadas can join the others. '
'Your armada has size: {0}.'.format(armada_size))
try:
agent_self_dict = consul_query('agent/self', consul_address='{0}:8500'.format(consul_host))
datacenter = agent_self_dict['Config']['Datacenter']
except:
return self.status_error('Could not read remote host datacenter address.')
current_consul_mode = _get_current_consul_mode()
if current_consul_mode == consul_config.ConsulMode.BOOTSTRAP:
override_runtime_settings(consul_mode=consul_config.ConsulMode.CLIENT,
ship_ips=[consul_host],
datacenter=datacenter)
else:
override_runtime_settings(ship_ips=[consul_host] + get_other_ship_ips(),
datacenter=datacenter)
if _restart_consul():
supervisor_server = xmlrpclib.Server('http://localhost:9001/RPC2')
hermes_init_output = supervisor_server.supervisor.startProcessGroup('hermes_init')
get_logger().info('hermes_init start: {}'.format(hermes_init_output))
set_ship_name(ship)
for key, data in local_services_data.items():
kv.kv_set(key, data)
return self.status_ok()
return self.status_error('Waiting for armada restart timed out.')
def set_ship_name(new_name):
ship_ip = get_ship_ip()
old_name = get_ship_name(ship_ip)
saved_containers = kv.kv_list('ships/{}/service/'.format(old_name))
if saved_containers:
for container in saved_containers:
new_key = 'ships/{}/service/{}/{}'.format(new_name, container.split('/')[-2], container.split('/')[-1])
container_dict = kv.kv_get(container)
kv.kv_set(new_key, container_dict)
kv.kv_remove(container)
kv.kv_set('ships/{}/name'.format(ship_ip), new_name)
kv.kv_set('ships/{}/ip'.format(new_name), ship_ip)
os.system('sed -i \'s|ships/{}/|ships/{}/|\' /etc/consul.config'.format(old_name, new_name))
try:
os.system('/usr/local/bin/consul reload')
except Exception as e:
get_logger().exception(e)
kv.kv_remove('containers_parameters_list/{}'.format(old_name))
def set_ship_name(new_name):
ship_ip = get_ship_ip()
old_name = get_ship_name(ship_ip)
saved_containers = kv.kv_list('ships/{}/service/'.format(old_name))
if saved_containers:
for container in saved_containers:
new_key = 'ships/{}/service/{}/{}'.format(new_name, container.split('/')[-2], container.split('/')[-1])
container_dict = kv.kv_get(container)
kv.kv_set(new_key, container_dict)
kv.kv_remove(container)
kv.kv_set('ships/{}/name'.format(ship_ip), new_name)
kv.kv_set('ships/{}/ip'.format(new_name), ship_ip)
os.system('sed -i \'s|ships/{}/|ships/{}/|\' /etc/consul.config'.format(old_name, new_name))
try:
os.system('/usr/local/bin/consul reload')
except Exception as e:
traceback.print_exc()
kv.kv_remove('containers_parameters_list/{}'.format(old_name))
def set_ship_name(new_name):
from armada_backend.models.services import get_services_by_ship, create_consul_services_key
ship_ip = get_ship_ip()
old_name = get_ship_name(ship_ip)
saved_containers = get_services_by_ship(old_name)
if saved_containers:
for container in saved_containers:
new_key = create_consul_services_key(
ship=new_name,
service_name=container.split('/')[-2],
container_id=container.split('/')[-1])
container_dict = kv.kv_get(container)
kv.kv_set(new_key, container_dict)
kv.kv_remove(container)
kv.kv_set('ships/{}/name'.format(ship_ip), new_name)
kv.kv_set('ships/{}/ip'.format(new_name), ship_ip)
os.system('sed -i \'s|ships/{}/|ships/{}/|\' /etc/consul.config'.format(
old_name, new_name))
try:
os.system('/usr/local/bin/consul reload')
except Exception as e:
get_logger().exception(e)
def set_initialized():
kv.kv_set('dockyard/initialized', '1')
def set_default(name):
kv.kv_set('dockyard/default', name)
def set_initialized():
kv.kv_set('dockyard/initialized', '1')
def set_default(name):
kv.kv_set('dockyard/default', name)
def update_service_dict(ship, service_name, container_id, key, value):
consul_key = create_consul_services_key(ship, service_name, container_id)
service_dict = kv_get(consul_key)
service_dict[key] = value
kv_set(consul_key, service_dict)
def set_ship_name(name):
ship_ip = get_ship_ip()
kv.kv_set('ships/{}/name'.format(ship_ip), name)
kv.kv_set('ships/{}/ip'.format(name), ship_ip)
def _save_containers_parameters_list_in_kv_store(containers_parameters_list):
ship_name = get_ship_name()
kv.kv_set('containers_parameters_list/{ship_name}'.format(**locals()),
containers_parameters_list)
def set_ship_name(name):
ship_ip = get_ship_ip()
kv.kv_set('ships/{}/name'.format(ship_ip), name)
kv.kv_set('ships/{}/ip'.format(name), ship_ip)
def _save_containers_parameters_list_in_kv_store(containers_parameters_list):
ship_name = get_ship_name()
kv.kv_set('containers_parameters_list/{ship_name}'.format(**locals()), containers_parameters_list)