def _choose_active_instances(services_dicts): result = services_dicts running_services_with_single_active_instances = {} for microservice_id, service_dict in services_dicts.items(): if service_dict.get('single_active_instance' ) and service_dict['status'] in ('passing', 'warning'): key = 'chosen_active_instance/{},env={},app_id={}'.format( service_dict['name'], service_dict['tags'].get('env') or '', service_dict['tags'].get('app_id') or '') if key not in running_services_with_single_active_instances: running_services_with_single_active_instances[key] = set() running_services_with_single_active_instances[key].add( microservice_id) for key, running_microservice_ids in running_services_with_single_active_instances.items( ): currently_picked_instance = kv.kv_get(key) if currently_picked_instance not in running_microservice_ids: currently_picked_instance = random.choice( list(running_microservice_ids)) kv.kv_set(key, currently_picked_instance) for microservice_id in running_microservice_ids: if microservice_id != currently_picked_instance: result[microservice_id]['status'] = 'standby' return result
def register_service_in_consul(microservice_data): if _exists_service(microservice_data['microservice_id']): return consul_service_data = { 'ID': microservice_data['microservice_id'], 'Name': microservice_data['microservice_name'], 'Port': microservice_data['microservice_port'], 'Check': { 'TTL': '15s', } } if microservice_data['microservice_tags']: consul_service_data['Tags'] = microservice_data['microservice_tags'] response = consul_post('agent/service/register', consul_service_data) response.raise_for_status() container_id = microservice_data['microservice_id'].split(':')[0] kv_set('start_timestamp/{}'.format(container_id), str(microservice_data['container_created_timestamp'])) key = 'single_active_instance/{}'.format(microservice_data['microservice_id']) if microservice_data.get('single_active_instance'): kv_set(key, True) else: kv_remove(key)
def save_container(ship, container_id, status, params=None): try: start_timestamp = kv_get('start_timestamp/{}'.format(container_id)) except: start_timestamp = None if status == 'crashed': service_name = params['microservice_name'] else: service_name = get_env(container_id, 'MICROSERVICE_NAME') params = json.loads( base64.b64decode( get_env(container_id, 'RESTART_CONTAINER_PARAMETERS'))) if not start_timestamp: start_timestamp = str(calendar.timegm(time.gmtime())) address = kv_get('ships/{}/ip'.format(ship)) or ship service_dict = { 'ServiceName': service_name, 'Status': status, 'container_id': container_id, 'params': params, 'start_timestamp': start_timestamp, 'ServiceID': container_id, 'Address': address } kv_set(create_consul_services_key(ship, service_name, container_id), service_dict)
def register_service_in_consul(microservice_data): if exists_service(microservice_data['microservice_id']): return consul_service_data = { 'ID': microservice_data['microservice_id'], 'Name': microservice_data['microservice_name'], 'Port': microservice_data['microservice_port'], 'Check': { 'TTL': '15s', } } microservice_tags = microservice_data.get('microservice_tags') if microservice_tags: consul_service_data['Tags'] = microservice_tags response = consul_post('agent/service/register', consul_service_data) response.raise_for_status() container_id = microservice_data['microservice_id'].split(':')[0] kv_set('start_timestamp/{}'.format(container_id), str(microservice_data['container_created_timestamp'])) key = 'single_active_instance/{}'.format( microservice_data['microservice_id']) if microservice_data.get('single_active_instance'): kv_set(key, True) else: kv_remove(key)
def update_container_status(status, ship=None, service_name=None, container_id=None, key=None): if not key: key = create_consul_services_key(ship, service_name, container_id) service_dict = kv_get(key) if status == 'crashed' and service_dict['Status'] in ['not-recovered', 'recovering']: return service_dict['Status'] = status kv_set(key, service_dict)
def _load_from_dict(saved_containers, containers_saved_in_kv, ship): for key, container_dict in saved_containers.items(): old_ship_name = key.split('/')[1] if old_ship_name != ship: key = 'ships/{}/service/{}/{}'.format(ship, container_dict['ServiceName'], container_dict['container_id']) if not containers_saved_in_kv or key not in containers_saved_in_kv: kv.kv_set(key, container_dict)
def set_alias(name, address, user=None, password=None): key = 'dockyard/aliases/{name}'.format(**locals()) value = {'address': address} if user: value['user'] = user if password: value['password'] = password will_be_default = len(get_list()) <= 1 and name != DOCKYARD_FALLBACK_ALIAS kv.kv_set(key, value) if will_be_default: set_default(name)
def on_post(self, req, resp): consul_host, error = self.get_post_parameter(req, 'host') if error: return self.status_error(resp, error) ship = get_ship_name() local_services_data = { key: kv.kv_get(key) for key in get_local_services_from_kv_store() } armada_size = _get_armada_size() if armada_size > 1: return self.status_error( resp, 'Currently only single ship armadas can join the others. ' 'Your armada has size: {0}.'.format(armada_size)) try: agent_self_dict = consul_query( 'agent/self', consul_address='{0}:8500'.format(consul_host)) datacenter = agent_self_dict['Config']['Datacenter'] except Exception as e: get_logger().exception(e) return self.status_error( resp, 'Could not read remote host datacenter address.') current_consul_mode = _get_current_consul_mode() if current_consul_mode == consul_config.ConsulMode.BOOTSTRAP: override_runtime_settings( consul_mode=consul_config.ConsulMode.CLIENT, ship_ips=[consul_host], datacenter=datacenter) else: override_runtime_settings(ship_ips=[consul_host] + get_other_ship_ips(), datacenter=datacenter) if _restart_consul(): supervisor_server = xmlrpc.client.Server( 'http://localhost:9001/RPC2') hermes_init_output = supervisor_server.supervisor.startProcessGroup( 'hermes_init') get_logger().info('hermes_init start: %s', hermes_init_output) set_ship_name(ship) for key, data in six.iteritems(local_services_data): kv.kv_set(key, data) return self.status_ok(resp) return self.status_error(resp, 'Waiting for armada restart timed out.')
def _choose_active_instances(services_dicts): result = services_dicts running_services_with_single_active_instances = {} for microservice_id, service_dict in services_dicts.items(): if service_dict.get('single_active_instance') and service_dict['status'] in ('passing', 'warning'): key = 'chosen_active_instance/{},env={},app_id={}'.format(service_dict['name'], service_dict['tags'].get('env') or '', service_dict['tags'].get('app_id') or '') if key not in running_services_with_single_active_instances: running_services_with_single_active_instances[key] = set() running_services_with_single_active_instances[key].add(microservice_id) for key, running_microservice_ids in running_services_with_single_active_instances.items(): currently_picked_instance = kv.kv_get(key) if currently_picked_instance not in running_microservice_ids: currently_picked_instance = random.choice(list(running_microservice_ids)) kv.kv_set(key, currently_picked_instance) for microservice_id in running_microservice_ids: if microservice_id != currently_picked_instance: result[microservice_id]['status'] = 'standby' return result
def POST(self): consul_host, error = self.get_post_parameter('host') if error: return self.status_error(error) ship = get_ship_name() local_services = kv.kv_list('ships/{}/service/'.format(ship)) or [] local_services_data = {key: kv.kv_get(key) for key in local_services} armada_size = _get_armada_size() if armada_size > 1: return self.status_error('Currently only single ship armadas can join the others. ' 'Your armada has size: {0}.'.format(armada_size)) try: agent_self_dict = consul_query('agent/self', consul_address='{0}:8500'.format(consul_host)) datacenter = agent_self_dict['Config']['Datacenter'] except: return self.status_error('Could not read remote host datacenter address.') current_consul_mode = _get_current_consul_mode() if current_consul_mode == consul_config.ConsulMode.BOOTSTRAP: override_runtime_settings(consul_mode=consul_config.ConsulMode.CLIENT, ship_ips=[consul_host], datacenter=datacenter) else: override_runtime_settings(ship_ips=[consul_host] + get_other_ship_ips(), datacenter=datacenter) if _restart_consul(): supervisor_server = xmlrpclib.Server('http://localhost:9001/RPC2') hermes_init_output = supervisor_server.supervisor.startProcessGroup('hermes_init') get_logger().info('hermes_init start: {}'.format(hermes_init_output)) set_ship_name(ship) for key, data in local_services_data.items(): kv.kv_set(key, data) return self.status_ok() return self.status_error('Waiting for armada restart timed out.')
def set_ship_name(new_name): ship_ip = get_ship_ip() old_name = get_ship_name(ship_ip) saved_containers = kv.kv_list('ships/{}/service/'.format(old_name)) if saved_containers: for container in saved_containers: new_key = 'ships/{}/service/{}/{}'.format(new_name, container.split('/')[-2], container.split('/')[-1]) container_dict = kv.kv_get(container) kv.kv_set(new_key, container_dict) kv.kv_remove(container) kv.kv_set('ships/{}/name'.format(ship_ip), new_name) kv.kv_set('ships/{}/ip'.format(new_name), ship_ip) os.system('sed -i \'s|ships/{}/|ships/{}/|\' /etc/consul.config'.format(old_name, new_name)) try: os.system('/usr/local/bin/consul reload') except Exception as e: get_logger().exception(e) kv.kv_remove('containers_parameters_list/{}'.format(old_name))
def set_ship_name(new_name): ship_ip = get_ship_ip() old_name = get_ship_name(ship_ip) saved_containers = kv.kv_list('ships/{}/service/'.format(old_name)) if saved_containers: for container in saved_containers: new_key = 'ships/{}/service/{}/{}'.format(new_name, container.split('/')[-2], container.split('/')[-1]) container_dict = kv.kv_get(container) kv.kv_set(new_key, container_dict) kv.kv_remove(container) kv.kv_set('ships/{}/name'.format(ship_ip), new_name) kv.kv_set('ships/{}/ip'.format(new_name), ship_ip) os.system('sed -i \'s|ships/{}/|ships/{}/|\' /etc/consul.config'.format(old_name, new_name)) try: os.system('/usr/local/bin/consul reload') except Exception as e: traceback.print_exc() kv.kv_remove('containers_parameters_list/{}'.format(old_name))
def set_ship_name(new_name): from armada_backend.models.services import get_services_by_ship, create_consul_services_key ship_ip = get_ship_ip() old_name = get_ship_name(ship_ip) saved_containers = get_services_by_ship(old_name) if saved_containers: for container in saved_containers: new_key = create_consul_services_key( ship=new_name, service_name=container.split('/')[-2], container_id=container.split('/')[-1]) container_dict = kv.kv_get(container) kv.kv_set(new_key, container_dict) kv.kv_remove(container) kv.kv_set('ships/{}/name'.format(ship_ip), new_name) kv.kv_set('ships/{}/ip'.format(new_name), ship_ip) os.system('sed -i \'s|ships/{}/|ships/{}/|\' /etc/consul.config'.format( old_name, new_name)) try: os.system('/usr/local/bin/consul reload') except Exception as e: get_logger().exception(e)
def set_initialized(): kv.kv_set('dockyard/initialized', '1')
def set_default(name): kv.kv_set('dockyard/default', name)
def update_service_dict(ship, service_name, container_id, key, value): consul_key = create_consul_services_key(ship, service_name, container_id) service_dict = kv_get(consul_key) service_dict[key] = value kv_set(consul_key, service_dict)
def set_ship_name(name): ship_ip = get_ship_ip() kv.kv_set('ships/{}/name'.format(ship_ip), name) kv.kv_set('ships/{}/ip'.format(name), ship_ip)
def _save_containers_parameters_list_in_kv_store(containers_parameters_list): ship_name = get_ship_name() kv.kv_set('containers_parameters_list/{ship_name}'.format(**locals()), containers_parameters_list)