def upgrade_node(upd, with_testing, env, *args, **kwargs):
upd.print_log('Change log template in rsyslog configuration...')
run("sed -i '/^{0}/d; a{0};{1}' {2}".format(PARAM, TEMPLATE, CONF))
run('systemctl restart rsyslog')
upd.print_log('Update logging pod...')
ki = User.filter_by(username=KUBERDOCK_INTERNAL_USER).first()
pod_name = get_kuberdock_logs_pod_name(env.host_string)
for pod in PodCollection(ki).get(as_json=False):
if pod['name'] == pod_name:
break
else:
return
PodCollection(ki).delete(pod['id'], force=True)
logs_config = get_kuberdock_logs_config(
env.host_string,
pod_name,
pod['kube_type'],
pod['containers'][0]['kubes'],
pod['containers'][1]['kubes'],
MASTER_IP,
ki.get_token(),
)
check_internal_pod_data(logs_config, user=ki)
logs_pod = PodCollection(ki).add(logs_config, skip_check=True)
run('docker pull kuberdock/fluentd:1.4')
PodCollection(ki).update(logs_pod['id'], {'command': 'start'})
def upgrade_node(upd, with_testing, env, *args, **kwargs):
upd.print_log('Add node hostname to rsyslog configuration...')
run("sed -i '/^{0}/d; i{0} {1}' {2}".format(PARAM, env.host_string, CONF))
run('systemctl restart rsyslog')
upd.print_log('Update logging pod...')
ki = User.filter_by(username=KUBERDOCK_INTERNAL_USER).first()
pod_name = get_kuberdock_logs_pod_name(env.host_string)
for pod in PodCollection(ki).get(as_json=False):
if pod['name'] == pod_name:
break
else:
return
PodCollection(ki).delete(pod['id'], force=True)
logs_config = get_kuberdock_logs_config(
env.host_string,
pod_name,
pod['kube_type'],
pod['containers'][0]['kubes'],
pod['containers'][1]['kubes'],
MASTER_IP,
ki.get_token(),
)
check_internal_pod_data(logs_config, user=ki)
logs_pod = PodCollection(ki).add(logs_config, skip_check=True)
run('rm -fr /var/lib/elasticsearch/kuberdock/nodes/*/indices/syslog-*')
PodCollection(ki).update(logs_pod['id'], {'command': 'start'})
def check_nodes():
msg = []
states = {}
try:
states = get_nodes_state()
for node, state in states.items():
node_msg = []
for key, state in state.items():
if not state or not isinstance(state, bool):
node_msg.append(MESSAGES[key].format(state))
if node_msg:
msg.append("Node {} errors:".format(node))
msg.extend(node_msg)
except (SystemExit, Exception) as e:
msg.append("Can't get nodes list because of {}".format(e.message))
pendings = [
get_kuberdock_logs_pod_name(node) for node, state in states.items()
if NODE_STATUSES.pending in state
]
if states and len(pendings) != len(states):
try:
pod_states = get_internal_pods_state()
stopped = [
pod for pod, status in pod_states.items()
if pod not in pendings and not status
]
if stopped:
msg.append(MESSAGES['pods'].format(', '.join(stopped)))
except (SystemExit, Exception) as e:
msg.append("Can't get internal pods states because of {}".format(
e.message))
return os.linesep.join(msg)
def _upgrade_node_202(upd, with_testing, env, *args, **kwargs):
"""Update log pod"""
upd.print_log("Upgrading logs pod ...")
ki = User.get_internal()
pod_name = nodes.get_kuberdock_logs_pod_name(env.host_string)
for pod in PodCollection(ki).get(as_json=False):
if pod['name'] == pod_name:
PodCollection(ki).delete(pod['id'], force=True)
break
else:
upd.print_log(u"Warning: logs pod '{}' not found".format(pod_name))
run('docker pull kuberdock/elasticsearch:2.2')
run('docker pull kuberdock/fluentd:1.8')
log_pod = nodes.create_logs_pod(env.host_string, ki)
# Also we should update rsyslog config, because log pod IP was changed.
pod_ip = log_pod['podIP']
put(StringIO('$LocalHostName {node_name}\n'
'$template LongTagForwardFormat,'
'"<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% '
'%syslogtag%%msg:::sp-if-no-1st-sp%%msg%"\n'
'*.* @{pod_ip}:5140;LongTagForwardFormat\n'.format(
node_name=env.host_string, pod_ip=pod_ip)),
RSYSLOG_CONF,
mode=0644)
run('systemctl restart rsyslog')
upd.print_log("Logs pod successfully upgraded")
def upgrade_node(upd, with_testing, env, *args, **kwargs):
run('yum --enablerepo=kube,kube-testing clean metadata')
# 00091_update.py
upd.print_log('Upgrading nodes with docker-cleaner.sh')
run("""rm -f /var/lib/kuberdock/scripts/docker-cleaner.sh""")
run("""crontab -l | grep -v "docker-cleaner.sh" | crontab - """)
# 00092_update.py
put('/var/opt/kuberdock/node_network_plugin.sh',
'/usr/libexec/kubernetes/kubelet-plugins/net/exec/kuberdock/kuberdock')
# 00093_update.py
upd.print_log('Use custom log template with rsyslog...')
run("sed -i '/^{0}/d' {1}".format(PARAM1, CONF))
run("sed -i '/^{0}/d' {1}".format(PARAM2, CONF))
run("sed -i '$ a{0} {1}' {2}".format(PARAM1, TEMPLATE, CONF))
run("sed -i '$ a{0};{1}' {2}".format(PARAM2, TEMPLATE_NAME, CONF))
run('systemctl restart rsyslog')
# 00096_update.py
upd.print_log('Disabling swap and backing up fstab to {0}...'.format(FSTAB_BACKUP))
run('swapoff -a')
run('mkdir -p /var/lib/kuberdock/backups')
run('test -f {0} && echo "{0} is already exists" || cp /etc/fstab {0}'.format(FSTAB_BACKUP))
run("sed -r -i '/[[:space:]]+swap[[:space:]]+/d' /etc/fstab")
# 00097_update.py
upd.print_log('Update elasticsearch for logs...')
upd.print_log(put('/var/opt/kuberdock/make_elastic_config.py',
'/var/lib/elasticsearch',
mode=0755))
upd.print_log('Update logging pod...')
ki = User.filter_by(username=KUBERDOCK_INTERNAL_USER).first()
pod_name = get_kuberdock_logs_pod_name(env.host_string)
for pod in PodCollection(ki).get(as_json=False):
if pod['name'] == pod_name:
break
else:
return
PodCollection(ki).delete(pod['id'], force=True)
logs_config = get_kuberdock_logs_config(
env.host_string,
pod_name,
pod['kube_type'],
pod['containers'][0]['kubes'],
pod['containers'][1]['kubes'],
MASTER_IP,
ki.get_token(),
)
check_internal_pod_data(logs_config, user=ki)
logs_pod = PodCollection(ki).add(logs_config, skip_check=True)
run('docker pull kuberdock/elasticsearch:2.2')
run('docker pull kuberdock/fluentd:1.5')
PodCollection(ki).update(logs_pod['id'], {'command': 'start'})
def upgrade_node(upd, with_testing, env, *args, **kwargs):
# 00076_update.py
upd.print_log('Add kernel parameters to make pod isolation work...')
run('sed -i "/net.bridge.bridge-nf-call-ip6\?tables/d" {0}'.format(CONF))
run("echo net.bridge.bridge-nf-call-iptables = 1 >> {0}".format(CONF))
run("echo net.bridge.bridge-nf-call-ip6tables = 1 >> {0}".format(CONF))
run("sysctl -w net.bridge.bridge-nf-call-iptables=1")
run("sysctl -w net.bridge.bridge-nf-call-ip6tables=1")
# 00079_update.py
upd.print_log('Copy Elasticsearch config maker...')
upd.print_log(
put('/var/opt/kuberdock/make_elastic_config.py',
'/var/lib/elasticsearch',
mode=0755))
upd.print_log('Update logging pod...')
ki = User.filter_by(username=KUBERDOCK_INTERNAL_USER).first()
pod_name = get_kuberdock_logs_pod_name(env.host_string)
for pod in PodCollection(ki).get(as_json=False):
if pod['name'] == pod_name:
break
else:
return
PodCollection(ki).delete(pod['id'], force=True)
logs_config = get_kuberdock_logs_config(
env.host_string,
pod_name,
pod['kube_type'],
pod['containers'][0]['kubes'],
pod['containers'][1]['kubes'],
MASTER_IP,
ki.get_token(),
)
check_internal_pod_data(logs_config, user=ki)
logs_pod = PodCollection(ki).add(logs_config, skip_check=True)
run('docker pull kuberdock/elasticsearch:1.5')
PodCollection(ki).update(logs_pod['id'], {'command': 'start'})
# 00082_update.py
upd.print_log('Upgrading nodes with docker-cleaner.sh')
run("cat > /var/lib/kuberdock/scripts/docker-cleaner.sh << 'EOF' {0}".
format(DOCKERCLEANER))
run("""chmod +x /var/lib/kuberdock/scripts/docker-cleaner.sh""")
run("""crontab -l | { cat; echo "0 */6 * * * /var/lib/kuberdock/scripts/docker-cleaner.sh"; } | crontab - """
)
def _update_00179_upgrade_node(env):
nodename = env.host_string
log_pod_name = get_kuberdock_logs_pod_name(nodename)
internal_user = User.get_internal()
pc = PodCollection(internal_user)
dbpod = Pod.query.filter(Pod.owner_id == internal_user.id,
Pod.name == log_pod_name,
Pod.status != 'deleted').first()
if not dbpod:
raise Exception('Node {} have no logs pod. '
'Delete the node and try again'.format(nodename))
pod = pc.get(dbpod.id, as_json=False)
old_ip = '127.0.0.1'
new_ip = pod['podIP']
run('sed -i "s/@{old_ip}:/@{new_ip}:/g" {conf}'.format(old_ip=old_ip,
new_ip=new_ip,
conf=RSYSLOG_CONF))
run('systemctl restart rsyslog')
def update_log_pods(user):
for pod in PodCollection(user).get(as_json=False):
if pod_name_pattern.match(pod['name']):
PodCollection(user).delete(pod['id'], force=True)
logs_kubes = 1
logcollector_kubes = logs_kubes
logstorage_kubes = logs_kubes
node_resources = kubes_to_limits(
logs_kubes, INTERNAL_SERVICE_KUBE_TYPE)['resources']
logs_memory_limit = node_resources['limits']['memory']
if logs_memory_limit < KUBERDOCK_LOGS_MEMORY_LIMIT:
logs_kubes = int(math.ceil(
float(KUBERDOCK_LOGS_MEMORY_LIMIT) / logs_memory_limit
))
if logs_kubes > 1:
# allocate total log cubes to log collector and to log
# storage/search containers as 1 : 3
total_kubes = logs_kubes * 2
logcollector_kubes = int(math.ceil(float(total_kubes) / 4))
logstorage_kubes = total_kubes - logcollector_kubes
for node in Node.query:
hostname = node.hostname
podname = get_kuberdock_logs_pod_name(hostname)
logs_config = get_kuberdock_logs_config(
hostname,
podname,
INTERNAL_SERVICE_KUBE_TYPE,
logcollector_kubes,
logstorage_kubes,
MASTER_IP,
user.token,
)
check_internal_pod_data(logs_config, user=user)
logs_pod = PodCollection(user).add(logs_config)
PodCollection(user).update(logs_pod['id'], {'command': 'start'})
def test_execute_es_query(self, es_mock):
"""Test elasticsearch_utils.execute_es_query function."""
# Add two log pods config + two approprate nodes
internal_user = User.get_internal()
pod_id1 = str(uuid4())
service1 = 'srv1'
namespace1 = 'ns1'
pod_id2 = str(uuid4())
service2 = 'srv2'
namespace2 = 'ns2'
host1 = 'h1'
host2 = 'h2'
kube_id = Kube.get_default_kube_type()
pod1 = Pod(id=pod_id1,
name=get_kuberdock_logs_pod_name(host1),
owner_id=internal_user.id,
kube_id=kube_id,
config=json.dumps({
"service": service1,
"namespace": namespace1
}),
status='RUNNING')
pod2 = Pod(id=pod_id2,
name=get_kuberdock_logs_pod_name(host2),
owner_id=internal_user.id,
kube_id=kube_id,
config=json.dumps({
"service": service2,
"namespace": namespace2
}),
status='RUNNING')
db.session.add_all([pod1, pod2])
db.session.commit()
node1 = Node(ip='123.123.123',
hostname=host1,
kube_id=kube_id,
state='completed',
upgrade_status='applied')
node2 = Node(ip='123.123.124',
hostname=host2,
kube_id=kube_id,
state='completed',
upgrade_status='applied')
db.session.add_all([node1, node2])
db.session.commit()
size = 123
index = '1234qwerty'
query = None
sort = None
search_result = {'hits': {'total': 333, 'hits': [1, 2, 3]}}
search_mock = es_mock.return_value.search
search_mock.return_value = search_result
res = elasticsearch_utils.execute_es_query(index, query, size, sort)
self.assertEqual(
res, {
'total': search_result['hits']['total'],
'hits': search_result['hits']['hits'],
})
prefix1 = elasticsearch_utils.K8S_PROXY_PREFIX + \
'/namespaces/' + namespace1 + '/services/' + service1 + ':9200/'
prefix2 = elasticsearch_utils.K8S_PROXY_PREFIX + \
'/namespaces/' + namespace2 + '/services/' + service2 + ':9200/'
es_mock.assert_called_once_with([
{
'host': KUBE_API_HOST,
'port': KUBE_API_PORT,
'url_prefix': prefix1,
},
{
'host': KUBE_API_HOST,
'port': KUBE_API_PORT,
'url_prefix': prefix2,
},
])
search_mock.assert_called_once_with(index=index, body={'size': size})
query = {'a': 1}
sort = {'b': 2}
elasticsearch_utils.execute_es_query(index, query, size, sort)
search_mock.assert_called_with(index=index,
body={
'size': size,
'sort': sort,
'query': query
})
search_mock.side_effect = RequestException('!!!')
with self.assertRaises(elasticsearch_utils.LogsError):
elasticsearch_utils.execute_es_query(index, query, size, sort)
# You should have received a copy of the GNU General Public License
# along with KuberDock; if not, see <http://www.gnu.org/licenses/>.
import re
import yaml
from kubedock.kapi.nodes import (
get_kuberdock_logs_config,
get_kuberdock_logs_pod_name,
)
from kubedock.kapi.podcollection import PodCollection
from kubedock.settings import KUBERDOCK_INTERNAL_USER, MASTER_IP
from kubedock.users.models import User
from kubedock.validation import check_new_pod_data
pod_name_pattern = re.compile(get_kuberdock_logs_pod_name('.+?'))
def upgrade(upd, with_testing, *args, **kwargs):
upd.print_log('Fix logging pods path...')
with open('/etc/kubernetes/configfile_for_nodes') as node_configfile:
node_config = yaml.load(node_configfile.read())
for user in node_config['users']:
token = user['user']['token']
if user['name'] == 'kubelet':
break
ki = User.filter_by(username=KUBERDOCK_INTERNAL_USER).first()
for pod in PodCollection(ki).get(as_json=False):
if pod_name_pattern.match(pod['name']):
PodCollection(ki).delete(pod['id'], force=True)
logs_config = get_kuberdock_logs_config(