Exemple #1
0
def get_load_balancers():
    load_balancer_configs = hermes.get_config('load-balancers.json')
    if not load_balancer_configs:
        load_balancer_configs = [{
            'type': 'main-haproxy',
            'env': os.environ.get('MICROSERVICE_ENV'),
        }]
    for load_balancer_config in load_balancer_configs:
        load_balancer_type = load_balancer_config['type']
        if load_balancer_type == 'haproxy':
            load_balancer = haproxy.Haproxy(load_balancer_config)
            stats_config = load_balancer_config.get('stats')
            load_balancer.configure_stats(stats_config)
            yield load_balancer
        elif load_balancer_type == 'main-haproxy':
            main_haproxies = get_matching_main_haproxies(load_balancer_config.get('env'))
            for main_haproxy in main_haproxies:
                load_balancer = haproxy.MainHaproxy(main_haproxy)
                haproxy_parameters = load_balancer_config.get('haproxy_parameters')
                load_balancer.override_haproxy_parameters(haproxy_parameters)
                stats_config = load_balancer_config.get('stats')
                load_balancer.configure_stats(stats_config)
                yield load_balancer
        else:
            print_err("Unknown load-balancer type: {load_balancer_type}".format(**locals()))
Exemple #2
0
def main():
    tags = {"environment": os.environ.get('MICROSERVICE_ENV')}
    client = Client(hermes.get_config('config.json', {}).get('sentry-url', ''),
                    auto_log_stacks=True,
                    tags=tags)
    _set_up_logger(client)

    thread = threading.Thread(target=_update_all)
    thread.start()

    urls = (
        '/gitlab_web_hook',
        GitLabWebHook.__name__,
        '/health',
        Health.__name__,
        '/update_from_git',
        UpdateFromGit.__name__,
        '/update_from_hermes_directory',
        UpdateFromHermesDirectory.__name__,
        '/update_all',
        UpdateAll.__name__,
        '/hermes_address',
        HermesAddress.__name__,
        '/update_hermes',
        UpdateHermes.__name__,
        '/',
        Index.__name__,
    )
    app = SentryApplication(client,
                            logging=True,
                            mapping=urls,
                            fvars=globals())

    app.run()
Exemple #3
0
def _create_all_sources():
    sources_config_dir = hermes.get_config_file_path('sources')
    sources_configs_keys = hermes.get_configs_keys('sources')
    result = []
    were_errors = False
    if not sources_configs_keys:
        logging.warning('sources_configs_keys is empty')
        return result, were_errors

    for source_config_key in sources_configs_keys:
        logging.debug('source_config_key: {}'.format(source_config_key))
        try:
            sources_dict = hermes.get_config(source_config_key)
            logging.debug('sources_dict: {}'.format(sources_dict))
            assert isinstance(sources_dict, list)
        except:
            logging.error(
                'Config {source_config_key} does not contain json with list of sources.'.format(**locals()))
            traceback.print_exc()
            were_errors = True
            continue

        for source_dict in sources_dict:
            try:
                sources = list(_create_sources_from_dict(source_dict, sources_config_dir))
                logging.debug('adding sources: {}'.format(sources))
                result.extend(sources)
            except:
                logging.error('Invalid source configuration:\n{}'.format(source_dict))
                traceback.print_exc()
                were_errors = True
    return result, were_errors
Exemple #4
0
def _create_all_sources():
    sources_config_dir = hermes.get_config_file_path('sources')
    sources_configs_keys = hermes.get_configs_keys('sources')
    result = []
    were_errors = False
    if not sources_configs_keys:
        logging.warning('sources_configs_keys is empty')
        return result, were_errors

    for source_config_key in sources_configs_keys:
        logging.debug('source_config_key: {}'.format(source_config_key))
        try:
            sources_dict = hermes.get_config(source_config_key)
            logging.debug('sources_dict: {}'.format(sources_dict))
            assert isinstance(sources_dict, list)
        except Exception as e:
            logging.exception(
                'Config {source_config_key} does not contain json with list of sources.'
                .format(**locals()))
            were_errors = True
            continue

        for source_dict in sources_dict:
            try:
                sources = list(
                    _create_sources_from_dict(source_dict, sources_config_dir))
                logging.debug('adding sources: {}'.format(sources))
                result.extend(sources)
            except Exception as e:
                logging.exception(
                    'Invalid source configuration:\n{}'.format(source_dict))
                were_errors = True
    return result, were_errors
def get_destinations_for_alias(destination_alias):
    destination_dicts = hermes.get_config('destinations.json')
    if destination_dicts is None:
        raise DestinationException('Could not find destinations.json.')
    if destination_alias not in destination_dicts:
        logging.error(
            'Destination alias {0} is not defined in destinations.json.'.
            format(destination_alias))
        return []
    destination_config_dir = os.path.dirname(
        hermes.get_config_file_path('destinations.json'))
    if isinstance(destination_dicts[destination_alias], dict):
        return [
            Destination(destination_dicts[destination_alias],
                        destination_config_dir)
        ]
    elif isinstance(destination_dicts[destination_alias], list):
        result = []
        for destination_dict in destination_dicts[destination_alias]:
            result.append(Destination(destination_dict,
                                      destination_config_dir))
        return result
    else:
        logging.error(
            'Destination definition for alias {0} is neither list nor dict.'.
            format(destination_alias))
    return []
Exemple #6
0
def _set_up_logger(sentry_client):
    config = hermes.get_config('config.json', {})
    log_level_from_config = str(config.get('log_level')).upper()
    log_level = getattr(logging, log_level_from_config, logging.INFO)
    logging.basicConfig(
        level=log_level,
        format='%(asctime)s %(name)s [%(levelname)s] - %(message)s')

    handler = SentryHandler(sentry_client, level=logging.WARNING)
    setup_logging(handler)
Exemple #7
0
def main():
    config = hermes.get_config('config.json', {})
    debug = config.get('debug', False)
    app = Application(
        (
            url('/', IndexHandler),
            url('/version_check', VersionCheckHandler),
        ),
        debug=debug
    )
    app.listen(80)
    IOLoop.instance().start()
def main():
    enable_pretty_logging()

    config = hermes.get_config('config.json', {})
    debug = config.get('debug', False)
    app = Application((
        url('/', IndexHandler),
        url('/version_check', VersionCheckHandler),
    ),
                      debug=debug)
    app.listen(80)
    IOLoop.instance().start()
Exemple #9
0
def setup_sentry():
    sentry_url = hermes.get_config('config.json').get('sentry_url', '')

    sentry_client = Client(sentry_url,
                           auto_log_stacks=True,
                           ignore_exceptions=sentry_ignore_exceptions,
                           )

    handler = SentryHandler(sentry_client, level=logging.WARNING)
    setup_logging(handler)

    return sentry_client
Exemple #10
0
def main():
    configs = get_config('ssh.json')
    if not configs:
        return
    exit_code = 0
    for config in configs:
        local_port = config['local_port']
        try:
            check_call(['nc', '-z', '127.0.0.1', str(local_port)])
        except CalledProcessError:
            sys.stderr.write(
                'WARNING: Local port {} for SSH tunnel is not open!\n'.format(
                    local_port))
            exit_code = 1
    sys.exit(exit_code)
Exemple #11
0
def create_magellan_config_from_file(file_name):
    config = hermes.get_config(file_name)

    if config:
        for microservice_name, configurations in config.items():
            if isinstance(configurations, list):
                for configuration in configurations:
                    configure_single_requirement(microservice_name,
                                                 **configuration)

            elif isinstance(configurations, dict):
                configure_single_requirement(microservice_name,
                                             **configurations)
    else:
        logging.warning('Empty dependency file: {}'.format(file_name))
Exemple #12
0
def main():
    microservice_name = os.environ.get('MICROSERVICE_NAME')
    if 'slave' in microservice_name:
        mode = 'slave'
        DEFAULT_CONFIG['slaveof'] = '127.0.0.1 10003'
    else:
        mode = 'master'

    config = DEFAULT_CONFIG
    config.update(hermes.get_config(f'redis.{mode}.conf.json'))
    shutil.copyfile('/tmp/redis-stable/redis.conf', REDIS_CONFIG_PATH)
    with open(REDIS_CONFIG_PATH, 'a') as f:
        armada_config_section = create_redis_conf_section_from_dict(config)
        f.write(armada_config_section)

    os.execl('/usr/local/bin/redis-server', 'redis-server', REDIS_CONFIG_PATH)
Exemple #13
0
def get_destinations_for_alias(destination_alias):
    destination_dicts = hermes.get_config('destinations.json')
    if destination_dicts is None:
        raise DestinationException('Could not find destinations.json.')
    if destination_alias not in destination_dicts:
        logging.error('Destination alias {0} is not defined in destinations.json.'.format(destination_alias))
        return []
    destination_config_dir = os.path.dirname(hermes.get_config_file_path('destinations.json'))
    if isinstance(destination_dicts[destination_alias], dict):
        return [Destination(destination_dicts[destination_alias], destination_config_dir)]
    elif isinstance(destination_dicts[destination_alias], list):
        result = []
        for destination_dict in destination_dicts[destination_alias]:
            result.append(Destination(destination_dict, destination_config_dir))
        return result
    else:
        logging.error('Destination definition for alias {0} is neither list nor dict.'.format(destination_alias))
    return []
Exemple #14
0
def get_load_balancers():
    load_balancer_configs = hermes.get_config('load-balancers.json')
    if not load_balancer_configs:
        load_balancer_configs = [{
            'type': 'main-haproxy',
            'env': os.environ.get('MICROSERVICE_ENV'),
        }]
    for load_balancer_config in load_balancer_configs:
        load_balancer_type = load_balancer_config['type']
        if load_balancer_type == 'haproxy':
            load_balancer = haproxy.Haproxy(load_balancer_config)
            yield load_balancer
        elif load_balancer_type == 'main-haproxy':
            main_haproxies = get_matching_main_haproxies(load_balancer_config.get('env'))
            for main_haproxy in main_haproxies:
                load_balancer_config.update(main_haproxy)
                load_balancer = haproxy.MainHaproxy(load_balancer_config)
                yield load_balancer
        else:
            print_err("Unknown load-balancer type: {load_balancer_type}".format(**locals()))
Exemple #15
0
def main():
    configs = hermes.get_config('ssh.json')
    if not configs:
        return
    processes = []
    for config in configs:
        key_path = hermes.get_config_file_path(config['key'])
        key_dest_path = '/tmp/{}.key'.format(
            hashlib.md5(key_path.encode()).hexdigest())
        shutil.copy(key_path, key_dest_path)
        os.chmod(key_dest_path, 0o600)

        autossh = 'autossh'
        command_params = (
            autossh,
            '-M 0',
            '-N',
            '-C',
            '-o',
            'ServerAliveInterval=60',
            '-o',
            'ServerAliveCountMax=3',
            '-o',
            'StrictHostKeyChecking=no',
            '-p {}'.format(config['proxy_port']),
            '-i',
            '{}'.format(key_dest_path),
            '{}@{}'.format(config['proxy_user'], config['proxy_host']),
            '-L 0.0.0.0:{}:{}:{}'.format(config['local_port'],
                                         config['destination_host'],
                                         config['destination_port']),
        )
        process = Popen(command_params)
        processes.append(process)
    for process in processes:
        process.wait()
    if processes:
        sys.exit(1)
Exemple #16
0
import uuid
import logging
from urllib.parse import urlencode

from armada import hermes
from tornado.gen import Task
from tornado.ioloop import IOLoop
from tornado.escape import json_decode
from tornado.web import RequestHandler, HTTPError
from tornado.httpclient import AsyncHTTPClient, HTTPError as HTTPClientError, HTTPRequest

from utils import status, StrictVerboseVersion


http_client = AsyncHTTPClient()
config = hermes.get_config('config.json')
logging.basicConfig(format='%(asctime)-15s %(message)s')


class IndexHandler(RequestHandler):
    def get(self):
        self.write(status())


class VersionCheckHandler(RequestHandler):
    async def get(self):
        client_version = self._validate_client_version()
        # send version to GA, fire and forget
        IOLoop.current().spawn_callback(self._collect_data, client_version, self.request.remote_ip)
        latest_version = await self._get_latest_version()
        data = {
Exemple #17
0
from functools import wraps
from datetime import datetime

import bottle
from armada import hermes
from bottle import request, response, HTTPError

sys.path.append('/opt/microservice/src')
from common.consul import consul_get

WELCOME_MESSAGE = '''Welcome to main-haproxy.
Did you mean to get here?
Perhaps your magellan is not properly configured or requested service is down.
'''

auth_config = hermes.get_config('auth_config.json', {})

STORE_STATS_FOR_DAYS = 7
STATS_PATH = '/tmp/stats'
AUTHORIZATION_TOKEN = auth_config.get('main_haproxy_auth_token')
RESTRICT_ACCESS = bool(AUTHORIZATION_TOKEN)


def _restrict_access(raw_token):
    unauthorized_error = HTTPError(401)
    try:
        match = re.match(r'Token (?P<token>\w+)', raw_token)
    except TypeError:
        raise unauthorized_error

    if match is None:
import uuid
import logging
from urllib.parse import urlencode
import sys

from armada import hermes
from tornado.gen import Task
from tornado.ioloop import IOLoop
from tornado.escape import json_decode
from tornado.web import RequestHandler, HTTPError
from tornado.httpclient import AsyncHTTPClient, HTTPError as HTTPClientError, HTTPRequest

from utils import status, StrictVerboseVersion

http_client = AsyncHTTPClient()
config = hermes.get_config('config.json')
logging.basicConfig(format='%(asctime)-15s %(message)s')


class IndexHandler(RequestHandler):
    def get(self):
        self.write(status())


class VersionCheckHandler(RequestHandler):
    async def get(self):
        client_version = self._validate_client_version()
        print("X-Forwarded-For: {}, client_version: {}".format(
            self.request.headers.get('X-Forwarded-For', ''), client_version),
              file=sys.stderr)
        # send version to GA, fire and forget
Exemple #19
0
from armada import hermes


def is_slave(microservice_name):
    return 'slave' in microservice_name


r = redis.StrictRedis(port=80)

if not r.ping():
    print("Didn't receive PONG.")
    exit(2)

microservice_name = os.environ.get('MICROSERVICE_NAME')
redis_info = r.info()
if is_slave(microservice_name) and redis_info['master_link_status'] == 'down':
    print("Can't connect to master.")
    exit(1)

# enable memory_usage healthcheck when redis don't evict keys
if redis_info['maxmemory_policy'] == 'noeviction':
    config = hermes.get_config('health-check.json')
    current_memory_usage = redis_info['used_memory'] / redis_info[
        'maxmemory'] * 100
    used_memory_warn_level_proc = config.get('used_memory_warn_level_proc')
    if used_memory_warn_level_proc and current_memory_usage >= used_memory_warn_level_proc:
        print(
            f"Current memory usage {current_memory_usage}% is above {used_memory_warn_level_proc}%."
        )
        exit(1)
def test_getting_simple_config():
    config = get_config('config.json')
    assert config == config_dev
Exemple #21
0
def _set_up_logger():
    config = hermes.get_config('config.json', {})
    log_level_from_config = str(config.get('log_level')).upper()
    log_level = getattr(logging, log_level_from_config, logging.INFO)
    logging.basicConfig(level=log_level)