def main(): configure_logging() config = get_config() if not contains_cloudwatch_logs_config(config): logging.info('Cloudwatch Logs Agent disabled by configuration') sys.exit(0) logging.info('Configuring Cloudwatch Logs Agent') # identity = {'region': 'eu-west-1', 'accountId': 123456, 'instanceId': 'i-123'} identity = boto.utils.get_instance_identity()['document'] environment = { 'observed_log_files': config.get('cloudwatch_logs'), 'application_id': config.get('application_id'), 'application_version': config.get('application_version'), 'region': identity['region'], 'account_id': identity['accountId'], 'instance_id': identity['instanceId'] } try: write_file('/var/awslogs/etc/aws.conf', render_template(AWS_CONFIG_TEMPLATE, environment)) write_file('/var/awslogs/etc/awslogs.conf', render_template(AWSLOGS_CONFIG_TEMPLATE, environment)) start_awslogs_service() logging.info('Successfully configured and started Cloudwatch Logs Agent') except Exception as e: logging.error('Failed to configure Cloudwatch Logs Agent') logging.exception(e) sys.exit(1)
def main(): configure_logging() config = get_config() telegraf_config = get_telegraf_config(config) if not telegraf_config: logging.info('Telegraf disabled by configuration') sys.exit(0) logging.info('Configuring Telegraf') # identity = {'region': 'eu-west-1', 'accountId': 123456, 'instanceId': 'i-123'} identity = boto.utils.get_instance_identity()['document'] environment = { 'application_id': config.get('application_id'), 'application_version': config.get('application_version'), 'region': identity['region'], 'instance_id': identity['instanceId'] } environment.update(telegraf_config) try: write_file('/etc/telegraf/telegraf.conf', render_template(CONFIG_TEMPLATE, environment)) start_telegraf_service() logging.info('Successfully configured and started Telegraf') sys.exit(0) except Exception as e: logging.error('Failed to configure Telegraf') logging.exception(e) sys.exit(0)
def main(): configure_logging() config = get_config() instance_logs_url = config.get('instance_logs_url') if not instance_logs_url: logging.warn('No endpoint for instance logs configured.') return # identity = {'region': 'eu-west-1', 'accountId': 123456, 'instanceId': 'i-123'} identity = boto.utils.get_instance_identity()['document'] region = identity['region'] account_id = identity['accountId'] instance_id = identity['instanceId'] boot_time = get_boot_time() is_shutdown = False if len(sys.argv) > 1: is_shutdown = sys.argv[1] == '--shutdown' while True: for fn in glob.glob('/var/log/audit.log.*.gz'): push_audit_log(config, instance_logs_url, account_id, region, instance_id, boot_time, fn) if is_shutdown: for fn in glob.glob('/var/log/audit.log'): push_audit_log(config, instance_logs_url, account_id, region, instance_id, boot_time, fn, compress=True) return time.sleep(60)
def main(): """Configure custom sysctl parameters If a sysctl section is present, add the valid parameters to sysctl and reloads. """ CUSTOM_SYSCTL_CONF = '/etc/sysctl.d/99-custom.conf' configure_logging() config = get_config() sysctl = config.get('sysctl') if sysctl is None: sys.exit(0) try: sysctl_entries = ['{} = {}'.format(key, value) for key, value in sysctl.items()] with open(CUSTOM_SYSCTL_CONF, 'w') as file: file.write('\n'.join(sysctl_entries)+'\n') logging.info('Successfully written sysctl parameters') except Exception as e: logging.error('Failed to write sysctl parameters') logging.exception(e) sys.exit(1) try: exitcode = subprocess.call(['/sbin/sysctl', '-p', CUSTOM_SYSCTL_CONF]) if exitcode != 0: logging.error('Reloading sysctl failed with exitcode {}'.format(exitcode)) sys.exit(1) logging.info('Successfully reloaded sysctl parameters') except Exception as e: logging.error('Failed to reload sysctl') logging.exception(e) sys.exit(1)
def main(): """Configure values for white-listed sysfs paths""" SYSFS_WHITELIST = ['/sys/kernel/mm/transparent_hugepage/enabled'] configure_logging() config = get_config() sysfs = config.get('sysfs') if sysfs is None: sys.exit(0) disallowed_paths = set(sysfs.keys()) - set(SYSFS_WHITELIST) if disallowed_paths: logging.error( 'You are not allowed to edit the sysfs path(s) {}'.format( list(disallowed_paths))) # Sanitize our dict first clean_sysfs = { key: value for (key, value) in sysfs.items() if key not in disallowed_paths } try: for key, value in clean_sysfs.items(): with open(key, 'w') as file: file.write(value + '\n') logging.info('Successfully written allowed sysfs paths') except Exception as e: logging.error('Failed to write sysfs paths') logging.exception(e) sys.exit(1)
def main(): configure_logging() config = get_config() if not contains_valid_dockercfg(config): sys.exit(0) logging.info('Writing dockercfg') try: path = os.path.expanduser('~/.dockercfg') write_file(path, json.dumps(config.get('dockercfg'))) directory = os.path.expanduser('~/.docker') if not os.path.exists(directory): os.makedirs(directory) path = os.path.expanduser('~/.docker/config.json') if os.path.exists(path): #load data = json.loads(open(path).read()) existing = config.get('dockercfg', {}) #merge data['auths'].update(existing) #write write_file(path, json.dumps(data)) else: write_file(path, json.dumps( { 'auths' : config.get('dockercfg', {}) })) logging.info('Successfully placed dockercfg') except Exception as e: logging.error('Failed to create dockercfg') logging.exception(e) sys.exit(1)
def main(): configure_logging() config = get_config() if not contains_valid_dockercfg(config): sys.exit(0) logging.info('Writing dockercfg') try: path = os.path.expanduser('~/.dockercfg') write_file(path, json.dumps(config.get('dockercfg'))) directory = os.path.expanduser('~/.docker') if not os.path.exists(directory): os.makedirs(directory) path = os.path.expanduser('~/.docker/config.json') if os.path.exists(path): # load data = json.loads(open(path).read()) existing = config.get('dockercfg', {}) # merge data['auths'].update(existing) # write write_file(path, json.dumps(data)) else: write_file(path, json.dumps({ 'auths': config.get('dockercfg', {}) })) logging.info('Successfully placed dockercfg') except Exception as e: logging.error('Failed to create dockercfg') logging.exception(e) sys.exit(1)
def get_scalyr_api_key(): ''' Read Scalyr API key from Taupage config and set in template file ''' main_config = get_config() config = main_config.get('logging') if config: scalyr_api_key = config.get('scalyr_account_key', False) else: scalyr_api_key = False if scalyr_api_key: # If scalyr_api_key starts with "aws:kms:" then decrypt key match_kms_key = re.search('aws:kms:', scalyr_api_key, re.IGNORECASE) if match_kms_key: scalyr_api_key = re.sub(r'aws:kms:', '', scalyr_api_key) try: scalyr_api_key = subprocess.check_output([ 'python3', '/opt/taupage/bin/decrypt-kms.py', scalyr_api_key ]).decode('UTF-8').strip() except Exception: logger.error('Failed to run /opt/taupage/bin/decrypt-kms.py') raise SystemExit() if scalyr_api_key == "Invalid KMS key.": logger.error('Failed to decrypt KMS Key') raise SystemExit() return scalyr_api_key
def main(): configure_logging() config = get_config() instance_logs_url = config.get('instance_logs_url') if not instance_logs_url: logging.warn('No endpoint for instance logs configured.') return # identity = {'region': 'eu-west-1', 'accountId': 123456, 'instanceId': 'i-123'} identity = boto.utils.get_instance_identity()['document'] region = identity['region'] account_id = identity['accountId'] instance_id = identity['instanceId'] boot_time = get_boot_time() is_shutdown = False if len(sys.argv) > 1: is_shutdown = sys.argv[1] == '--shutdown' while True: for fn in glob.glob('/var/log/audit.log.*.gz'): push_audit_log(config, instance_logs_url, account_id, region, instance_id, boot_time, fn) if is_shutdown: for fn in glob.glob('/var/log/audit.log'): push_audit_log(config, instance_logs_url, account_id, region, instance_id, boot_time, fn, compress=True) return rtime = random.randrange(60, 3000) time.sleep(rtime)
def main(): """Configure values for white-listed sysfs paths""" SYSFS_WHITELIST = ['/sys/kernel/mm/transparent_hugepage/enabled'] configure_logging() config = get_config() sysfs = config.get('sysfs') if sysfs is None: sys.exit(0) disallowed_paths = set(sysfs.keys()) - set(SYSFS_WHITELIST) if disallowed_paths: logging.error('You are not allowed to edit the sysfs path(s) {}'.format(list(disallowed_paths))) # Sanitize our dict first clean_sysfs = {key: value for (key, value) in sysfs.items() if key not in disallowed_paths} try: for key, value in clean_sysfs.items(): with open(key, 'w') as file: file.write(value + '\n') logging.info('Successfully written allowed sysfs paths') except Exception as e: logging.error('Failed to write sysfs paths') logging.exception(e) sys.exit(1)
def main(): configure_logging() config = get_config() instance_logs_url = config.get('instance_logs_url') if instance_logs_url: userAndPass = b64encode( bytes('{}:{}'.format(config.get('logsink_username'), config.get('logsink_password')), encoding='ascii')).decode("ascii") or '' # identity = {'region': 'eu-west-1', 'accountId': 123456, 'instanceId': 'i-123'} identity = boto.utils.get_instance_identity()['document'] region = identity['region'] account_id = identity['accountId'] instance_id = identity['instanceId'] boot_time = get_boot_time() # remove "sensitive" information from Taupage Config # (should be encrypted anyway, but better be sure..) masked_config = mask_dictionary(config) data = { 'account_id': str(account_id), 'region': region, 'instance_boot_time': boot_time, 'instance_id': instance_id, 'log_data': codecs.encode( yaml.safe_dump(masked_config).encode('utf-8'), 'base64').decode('utf-8'), 'log_type': 'USER_DATA' } logging.info('Pushing Taupage YAML to {}..'.format(instance_logs_url)) try: # TODO: use OAuth credentials response = requests.post(instance_logs_url, data=json.dumps(data), timeout=5, headers={ 'Content-Type': 'application/json', 'Authorization': 'Basic {}'.format(userAndPass) }) if response.status_code != 201: logging.warn( 'Failed to push Taupage YAML: server returned HTTP status {}: {}' .format(response.status_code, response.text)) except Exception: logging.exception('Failed to push Taupage YAML')
def main(): """Configure custom sysctl parameters If a sysctl section is present, add the valid parameters to sysctl and reloads. As some kernel parameters may not be allowed to be tuned, only parameters on a whitelist are allowed to be specified. """ SYSCTL_WHITELIST = [ 'fs.file-max', 'vm.dirty_background_ratio', 'vm.dirty_ratio', 'vm.overcommit_memory', 'vm.overcommit_ratio', 'vm.swappiness' ] CUSTOM_SYSCTL_CONF = '/etc/sysctl.d/99-custom.conf' configure_logging() config = get_config() sysctl = config.get('sysctl') if sysctl is None: sys.exit(0) disallowed_keys = set(sysctl.keys()) - set(SYSCTL_WHITELIST) if disallowed_keys: logging.error( 'You are not allowed to configure the sysctl parameters {}'.format( list(disallowed_keys))) try: sysctl_entries = [ '{} = {}'.format(key, value) for key, value in sysctl.items() if key in SYSCTL_WHITELIST ] with open(CUSTOM_SYSCTL_CONF, 'w') as file: file.write('\n'.join(sysctl_entries) + '\n') logging.info('Successfully written sysctl parameters') except Exception as e: logging.error('Failed to write sysctl parameters') logging.exception(e) sys.exit(1) try: exitcode = subprocess.call(['/sbin/sysctl', '-p', CUSTOM_SYSCTL_CONF]) if exitcode != 0: logging.error( 'Reloading sysctl failed with exitcode {}'.format(exitcode)) sys.exit(1) logging.info('Successfully reloaded sysctl parameters') except Exception as e: logging.error('Failed to reload sysctl') logging.exception(e) sys.exit(1)
def main(): configure_logging() config = get_config() if not contains_local_monitor_config(config): sys.exit(0) try: logging.info('Starting local_monitor') start_local_monitor_service() sys.exit(0) except Exception as e: logging.exception(e) sys.exit(0)
def main(): """Configure custom sysctl parameters If a sysctl section is present, add the valid parameters to sysctl and reloads. As some kernel parameters may not be allowed to be tuned, only parameters on a whitelist are allowed to be specified. """ SYSCTL_WHITELIST = ['fs.file-max', 'vm.dirty_background_ratio', 'vm.dirty_ratio', 'vm.max_map_count', 'vm.overcommit_memory', 'vm.overcommit_ratio', 'vm.swappiness', 'net.core.somaxconn'] CUSTOM_SYSCTL_CONF = '/etc/sysctl.d/99-custom.conf' configure_logging() config = get_config() sysctl = config.get('sysctl') if sysctl is None: sys.exit(0) disallowed_keys = set(sysctl.keys()) - set(SYSCTL_WHITELIST) if disallowed_keys: logging.error('You are not allowed to configure the sysctl parameters {}'.format(list(disallowed_keys))) try: sysctl_entries = ['{} = {}'.format(key, value) for key, value in sysctl.items() if key in SYSCTL_WHITELIST] with open(CUSTOM_SYSCTL_CONF, 'w') as file: file.write('\n'.join(sysctl_entries)+'\n') logging.info('Successfully written sysctl parameters') except Exception as e: logging.error('Failed to write sysctl parameters') logging.exception(e) sys.exit(1) try: exitcode = subprocess.call(['/sbin/sysctl', '-p', CUSTOM_SYSCTL_CONF]) if exitcode != 0: logging.error('Reloading sysctl failed with exitcode {}'.format(exitcode)) sys.exit(1) logging.info('Successfully reloaded sysctl parameters') except Exception as e: logging.error('Failed to reload sysctl') logging.exception(e) sys.exit(1)
def main(): # Process arguments args = process_arguments() if args.debug: configure_logging(logging.DEBUG) else: configure_logging(logging.INFO) # Load configuration from YAML file config = get_config(args.filename) if config.get("volumes"): handle_volumes(args, config) # Iterate over mount points iterate_mounts(config)
def main(): configure_logging() config = get_config() if not contains_valid_dockercfg(config): sys.exit(0) logging.info('Writing dockercfg') try: path = os.path.expanduser('~/.dockercfg') write_file(path, json.dumps(config.get('dockercfg'))) logging.info('Successfully placed dockercfg') except Exception as e: logging.error('Failed to create dockercfg') logging.exception(e) sys.exit(1)
def main(): configure_logging() config = get_config() instance_logs_url = config.get('instance_logs_url') if instance_logs_url: userAndPass = b64encode(bytes('{}:{}'.format( config.get('logsink_username'), config.get('logsink_password')), encoding='ascii')).decode("ascii") or '' # identity = {'region': 'eu-west-1', 'accountId': 123456, 'instanceId': 'i-123'} identity = boto.utils.get_instance_identity()['document'] region = identity['region'] account_id = identity['accountId'] instance_id = identity['instanceId'] boot_time = get_boot_time() # remove "sensitive" information from Taupage Config # (should be encrypted anyway, but better be sure..) masked_config = mask_dictionary(config) data = {'account_id': str(account_id), 'region': region, 'instance_boot_time': boot_time, 'instance_id': instance_id, 'log_data': codecs.encode(yaml.safe_dump(masked_config).encode('utf-8'), 'base64').decode('utf-8'), 'log_type': 'USER_DATA'} logging.info('Pushing Taupage YAML to {}..'.format(instance_logs_url)) try: # TODO: use OAuth credentials response = requests.post(instance_logs_url, data=json.dumps(data), timeout=5, headers={'Content-Type': 'application/json', 'Authorization': 'Basic {}'.format(userAndPass)}) if response.status_code != 201: logging.warn('Failed to push Taupage YAML: server returned HTTP status {}: {}'.format( response.status_code, response.text)) except: logging.exception('Failed to push Taupage YAML')
def main(): """Configure custom sysctl parameters If a sysctl section is present, add the valid parameters to sysctl and reloads. """ CUSTOM_SYSCTL_CONF = '/etc/sysctl.d/99-custom.conf' configure_logging() config = get_config() sysctl = config.get('sysctl') if sysctl is None: sys.exit(0) try: sysctl_entries = [ '{} = {}'.format(key, value) for key, value in sysctl.items() ] with open(CUSTOM_SYSCTL_CONF, 'w') as file: file.write('\n'.join(sysctl_entries) + '\n') logging.info('Successfully written sysctl parameters') except Exception as e: logging.error('Failed to write sysctl parameters') logging.exception(e) sys.exit(1) try: exitcode = subprocess.call(['/sbin/sysctl', '-p', CUSTOM_SYSCTL_CONF]) if exitcode != 0: logging.error( 'Reloading sysctl failed with exitcode {}'.format(exitcode)) sys.exit(1) logging.info('Successfully reloaded sysctl parameters') except Exception as e: logging.error('Failed to reload sysctl') logging.exception(e) sys.exit(1)
self.logger.info("Waiting for instance {0} to become healthy in elb {1}".format(instance_id, elb_name)) for i in range(0, max(int(self.TIMEOUT / self.INTERVAL), 1)): state = self._get_elb_instance_state(instance_id, elb_name) if state == 'InService': self.logger.info("instance in service") return True else: self.logger.debug('waiting for instance') sleep(self.INTERVAL) self.logger.warning("timeout for in-service check exceeded") return False if __name__ == '__main__': region = get_instance_identity()['document']['region'] instance_id = get_instance_identity()['document']['instanceId'] config = get_config() loadbalancer_name = config['healthcheck']['loadbalancer_name'] healthchecker = ElbHealthChecker(region) is_in_service = healthchecker.is_in_service_from_elb_perspective(instance_id, loadbalancer_name) if is_in_service: sys.exit(0) else: sys.exit(1)
def main(): args = process_arguments() # Setup logging if args.debug: configure_logging(logging.DEBUG) else: configure_logging(logging.INFO) current_region = args.region if args.region else get_region() # Load configuration from YAML file config = get_config(args.filename) if config.get("network_interfaces"): handle_network_interfaces(current_region, config) # The goal here is to be able to assign static IPs to instances # Within the Zalando AWS account setup we have a private subnet per # AZ. The idea is to create an ENI in each AZ in the private subnet where # you want a static IP. This means, your instance is going to have two # network interfaces on the same subnet, which causes some issues. # # The below code is based of the explaination at: https://goo.gl/2D8KrV # for handling two network interfaces in the same subnet. # Setting this to 1 Allows you to have multiple network interfaces on the same # subnet, and have the ARPs for each interface be answered based # on whether or not the kernel would route a packet from the # the ARP'd IP out that interface with open("/proc/sys/net/ipv4/conf/all/arp_filter", "w") as all_arp_filter: all_arp_filter.write("1") network_interfaces = [] default_gateway = netifaces.gateways()['default'][netifaces.AF_INET][0] for device_index in range(0, len(config.get("network_interfaces")) + 1): network_interfaces.append("eth{}".format(device_index)) # Run dhclient on all newly created interfaces to enable them to get IPs # Note, we do not run dhclient on eth0 as this may affect network connectivity # of the instance for network_interface in network_interfaces[1:]: subprocess.check_call(["dhclient", str(network_interface)]) route_tables = [] # Here we implement source-based routing, according to the serverfault post linked above for device_index in range(0, len(config.get("network_interfaces")) + 1): route_tables.append("{} eth{}".format( device_index + 1, device_index)) with open("/etc/iproute2/rt_tables", "w") as rt_tables: rt_tables.write("\n".join(route_tables)) for network_interface in network_interfaces: interface = netifaces.ifaddresses(network_interface)[ netifaces.AF_INET][0] ip = interface['addr'] subnet_cidr = str(IPAddress(interface["netmask"]).netmask_bits()) subprocess.check_call(["ip", "route", "add", "default", "via", default_gateway, "dev", network_interface, "table", network_interface]) subprocess.check_call(["ip", "route", "add", subnet_cidr, "dev", network_interface, "src", ip, "table", network_interface]) subprocess.check_call(["ip", "rule", "add", "from", ip, "table", network_interface])
def main(): """Confugure custom routing if necessary""" configure_logging() config = get_config() nat_gateways = config.get('nat_gateways') if not nat_gateways or not isinstance( nat_gateways, dict): # nat gateways must be non empty dictionary sys.exit(0) METADATA_URL = 'http://169.254.169.254/latest/meta-data/' try: r = requests.get(METADATA_URL + 'placement/availability-zone') region = r.text.strip()[:-1] logging.info('Region=%s', region) r = requests.get(METADATA_URL + 'mac') mac = r.text.strip() r = requests.get(METADATA_URL + 'network/interfaces/macs/' + mac + '/subnet-id') subnet = r.text if subnet not in nat_gateways: logging.warning( 'Can not find subnet %s in the nat_gateways mapping', subnet) sys.exit(0) logging.info('Will use %s nat gateway for outgoing https traffic', nat_gateways[subnet]) except Exception: logging.exception('Failed to read metadata') sys.exit(1) RT_TABLES = '/etc/iproute2/rt_tables' try: with open(RT_TABLES, 'a') as f: f.write('\n150 https\n') logging.info('Created new routing table for https traffic') except Exception: logging.exception('Failed to write into %s', RT_TABLES) sys.exit(1) iptables = ['iptables', '-w', '-t', 'mangle'] subprocess_call(iptables + [ '-A', 'OUTPUT', '-p', 'tcp', '!', '-d', '172.16.0.0/12', '--dport', '443', '-j', 'MARK', '--set-mark', '443' ]) subprocess_call(iptables + [ '-A', 'OUTPUT', '-p', 'udp', '--dport', '123', '-j', 'MARK', '--set-mark', '443' ]) subprocess_call(['ip', 'rule', 'add', 'fwmark', '443', 'lookup', 'https']) subprocess_call([ 'ip', 'route', 'add', 'default', 'via', nat_gateways[subnet], 'table', 'https' ]) # S3 is exceptional, it has it's own endpoint in VPC try: r = requests.get('https://ip-ranges.amazonaws.com/ip-ranges.json') ranges = [ e['ip_prefix'] for e in r.json()['prefixes'] if e['service'] == 'S3' and e['region'] == region and 'ip_prefix' in e ] except Exception: logging.exception('Failed to load ip-ranges.json') # Don't mark outgoing traffic to S3 for r in ranges: subprocess_call(iptables + ['-I', 'OUTPUT', '-d', r, '-j', 'ACCEPT'])
def update_configuration_from_template(): ''' Update Jinja Template to create configuration file for Scalyr ''' fluentd_destinations = dict(scalyr=False, s3=False, rsyslog=False, scalyr_s3=False) config = get_config() logging_config = config.get('logging') application_id = config.get('application_id') application_version = config.get('application_version') stack = config.get('notify_cfn')['stack'] source = config.get('source') image = config.get('source').split(':', 1)[0] instance_data = boto.utils.get_instance_identity()['document'] aws_region = instance_data['region'] aws_account = instance_data['accountId'] hostname = boto.utils.get_instance_metadata()['local-hostname'].split( '.')[0] customlog = config.get('mount_custom_log') if config.get('rsyslog_aws_metadata'): scalyr_syslog_log_parser = 'systemLogMetadata' else: scalyr_syslog_log_parser = 'systemLog' scalyr_application_log_parser = logging_config.get( 'scalyr_application_log_parser', 'slf4j') scalyr_custom_log_parser = logging_config.get('scalyr_custom_log_parser', 'slf4j') fluentd_log_destination = logging_config.get('log_destination', 'scalyr') fluentd_syslog_destination = logging_config.get('syslog_destination', fluentd_log_destination) fluentd_applog_destination = logging_config.get('applog_destination', fluentd_log_destination) fluentd_authlog_destination = logging_config.get('authlog_destination', fluentd_log_destination) fluentd_customlog_destination = logging_config.get( 'customlog_destination', fluentd_log_destination) fluentd_loglevel = logging_config.get('fluentd_loglevel', 'info') fluentd_s3_region = logging_config.get('s3_region', 'eu-central-1') fluentd_s3_bucket = logging_config.get('s3_bucket') fluentd_s3_timekey = logging_config.get('s3_timekey', '1m') fluentd_rsyslog_host = logging_config.get('rsyslog_host') fluentd_rsyslog_port = logging_config.get('rsyslog_port', '514') fluentd_rsyslog_protocol = logging_config.get('rsyslog_protocol', 'tcp') fluentd_rsyslog_severity = logging_config.get('rsyslog_severity', 'notice') fluentd_rsyslog_program = logging_config.get('rsyslog_program', 'fluentd') fluentd_rsyslog_hostname = logging_config.get('rsyslog_hostname', hostname) for destination in (fluentd_applog_destination, fluentd_authlog_destination, fluentd_customlog_destination, fluentd_syslog_destination): fluentd_destinations[destination] = True # Get Scalyr key only if configured if fluentd_destinations.get('scalyr') or fluentd_destinations.get( 'scalyr_s3'): scalyr_api_key = get_scalyr_api_key() env = Environment(loader=FileSystemLoader(TD_AGENT_TEMPLATE_PATH), trim_blocks=True) template_data = env.get_template(TPL_NAME).render( scalyr_api_key=scalyr_api_key, application_id=application_id, application_version=application_version, stack=stack, source=source, image=image, aws_region=aws_region, aws_account=aws_account, customlog=customlog, scalyr_application_log_parser=scalyr_application_log_parser, scalyr_syslog_log_parser=scalyr_syslog_log_parser, scalyr_custom_log_parser=scalyr_custom_log_parser, fluentd_syslog_destination=fluentd_syslog_destination, fluentd_applog_destination=fluentd_applog_destination, fluentd_authlog_destination=fluentd_authlog_destination, fluentd_customlog_destination=fluentd_customlog_destination, fluentd_loglevel=fluentd_loglevel, fluentd_s3_region=fluentd_s3_region, fluentd_s3_bucket=fluentd_s3_bucket, fluentd_s3_timekey=fluentd_s3_timekey, fluentd_rsyslog_host=fluentd_rsyslog_host, fluentd_rsyslog_port=fluentd_rsyslog_port, fluentd_rsyslog_protocol=fluentd_rsyslog_protocol, fluentd_rsyslog_severity=fluentd_rsyslog_severity, fluentd_rsyslog_program=fluentd_rsyslog_program, fluentd_rsyslog_hostname=fluentd_rsyslog_hostname, fluentd_destinations=fluentd_destinations) try: with open(TD_AGENT_OUTPUT_PATH, 'w') as f: f.write(template_data) except Exception: logger.exception('Failed to write file td-agent.conf')
fluentd_rsyslog_program=fluentd_rsyslog_program, fluentd_rsyslog_hostname=fluentd_rsyslog_hostname, fluentd_destinations=fluentd_destinations) try: with open(TD_AGENT_OUTPUT_PATH, 'w') as f: f.write(template_data) except Exception: logger.exception('Failed to write file td-agent.conf') raise SystemExit(1) if __name__ == '__main__': hostname = boto.utils.get_instance_metadata()['local-hostname'].split( '.')[0] config = get_config() logging_config = config.get('logging') s3_default = False if logging_config: if not logging_config.get('fluentd_enabled'): logger.info('Fluentd disabled; skipping Fluentd initialization') raise SystemExit() if not logging_config: logger.info( 'Found no logging section in senza.yaml; enable dafault logging to s3' ) s3_default = True try: with open('/var/local/textfile_collector/fluentd_default_s3.prom', 'w') as file: file.write(
def update_configuration_from_template(s3_default): ''' Update Jinja Template to create configuration file for Scalyr ''' fluentd_destinations = dict(scalyr=False, s3=False, rsyslog=False, scalyr_s3=False) config = get_config() logging_config = config.get('logging', {}) application_id = config.get('application_id') application_version = config.get('application_version') stack = config.get('notify_cfn', {}).get('stack') source = config.get('source') image = config.get('source').split(':', 1)[0] instance_data = boto.utils.get_instance_identity()['document'] aws_region = instance_data['region'] aws_account = instance_data['accountId'] hostname = boto.utils.get_instance_metadata()['local-hostname'].split( '.')[0] customlog = config.get('mount_custom_log') if config.get('rsyslog_aws_metadata'): scalyr_syslog_log_parser = 'systemLogMetadata' else: scalyr_syslog_log_parser = 'systemLog' scalyr_application_log_parser = logging_config.get( 'scalyr_application_log_parser', 'slf4j') scalyr_custom_log_parser = logging_config.get('scalyr_custom_log_parser', 'slf4j') fluentd_log_destination = logging_config.get('log_destination', 's3') fluentd_syslog_destination = logging_config.get('syslog_destination', fluentd_log_destination) fluentd_applog_destination = logging_config.get('applog_destination', fluentd_log_destination) fluentd_authlog_destination = logging_config.get('authlog_destination', fluentd_log_destination) fluentd_customlog_destination = logging_config.get( 'customlog_destination', fluentd_log_destination) fluentd_applog_filter_exclude = logging_config.get('applog_filter_exclude', None) fluentd_customlog_filter_exclude = logging_config.get( 'customlog_filter_exclude', None) fluentd_loglevel = logging_config.get('fluentd_loglevel', 'error') fluentd_s3_raw_log_format = logging_config.get('s3_raw_log_format', 'true') fluentd_s3_region = logging_config.get('s3_region', aws_region) fluentd_s3_bucket = logging_config.get( 's3_bucket', 'zalando-logging-' + aws_account + '-' + aws_region) fluentd_s3_timekey = logging_config.get('s3_timekey', '5m') fluentd_s3_acl = logging_config.get('s3_acl', 'bucket-owner-full-control') fluentd_rsyslog_host = logging_config.get('rsyslog_host') fluentd_rsyslog_port = logging_config.get('rsyslog_port', '514') fluentd_rsyslog_protocol = logging_config.get('rsyslog_protocol', 'tcp') fluentd_rsyslog_severity = logging_config.get('rsyslog_severity', 'notice') fluentd_rsyslog_program = logging_config.get('rsyslog_program', 'fluentd') fluentd_rsyslog_hostname = logging_config.get('rsyslog_hostname', hostname) for destination in (fluentd_applog_destination, fluentd_authlog_destination, fluentd_customlog_destination, fluentd_syslog_destination): fluentd_destinations[destination] = True # Get Scalyr key only if configured if fluentd_destinations.get('scalyr') or fluentd_destinations.get( 'scalyr_s3'): scalyr_api_key = get_scalyr_api_key() else: scalyr_api_key = None if fluentd_destinations.get('s3') or fluentd_destinations.get('scalyr_s3'): try: with open('/etc/cron.d/s3-iam-check', 'w') as file: file.write('#!/bin/bash\n') file.write( 'PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\n' ) file.write( '*/5 * * * * root /opt/taupage/bin/s3-iam-check.py test {!s}\n' .format(fluentd_s3_bucket)) except Exception: logger.exception('Failed to write file /etc/cron.d/s3-iam-check') raise SystemExit(1) env = Environment(loader=FileSystemLoader(TD_AGENT_TEMPLATE_PATH), trim_blocks=True) template_data = env.get_template(TPL_NAME).render( scalyr_api_key=scalyr_api_key, application_id=application_id, application_version=application_version, stack=stack, source=source, image=image, aws_region=aws_region, aws_account=aws_account, customlog=customlog, scalyr_application_log_parser=scalyr_application_log_parser, scalyr_syslog_log_parser=scalyr_syslog_log_parser, scalyr_custom_log_parser=scalyr_custom_log_parser, fluentd_syslog_destination=fluentd_syslog_destination, fluentd_applog_destination=fluentd_applog_destination, fluentd_applog_filter_exclude=fluentd_applog_filter_exclude, fluentd_authlog_destination=fluentd_authlog_destination, fluentd_customlog_destination=fluentd_customlog_destination, fluentd_customlog_filter_exclude=fluentd_customlog_filter_exclude, fluentd_loglevel=fluentd_loglevel, fluentd_s3_raw_log_format=fluentd_s3_raw_log_format, fluentd_s3_region=fluentd_s3_region, fluentd_s3_bucket=fluentd_s3_bucket, fluentd_s3_timekey=fluentd_s3_timekey, fluentd_s3_acl=fluentd_s3_acl, fluentd_rsyslog_host=fluentd_rsyslog_host, fluentd_rsyslog_port=fluentd_rsyslog_port, fluentd_rsyslog_protocol=fluentd_rsyslog_protocol, fluentd_rsyslog_severity=fluentd_rsyslog_severity, fluentd_rsyslog_program=fluentd_rsyslog_program, fluentd_rsyslog_hostname=fluentd_rsyslog_hostname, fluentd_destinations=fluentd_destinations) try: with open(TD_AGENT_OUTPUT_PATH, 'w') as f: f.write(template_data) except Exception: logger.exception('Failed to write file td-agent.conf') raise SystemExit(1)
def main(): """Confugure custom routing if necessary""" configure_logging() config = get_config() nat_gateways = config.get('nat_gateways') if not nat_gateways or not isinstance(nat_gateways, dict): # nat gateways must be non empty dictionary sys.exit(0) METADATA_URL = 'http://169.254.169.254/latest/meta-data/' try: r = requests.get(METADATA_URL + 'placement/availability-zone') region = r.text.strip()[:-1] logging.info('Region=%s', region) r = requests.get(METADATA_URL + 'mac') mac = r.text.strip() r = requests.get(METADATA_URL + 'network/interfaces/macs/' + mac + '/subnet-id') subnet = r.text if subnet not in nat_gateways: logging.warning('Can not find subnet %s in the nat_gateways mapping', subnet) sys.exit(0) logging.info('Will use %s nat gateway for outgoing https traffic', nat_gateways[subnet]) except Exception: logging.exception('Failed to read metadata') sys.exit(1) RT_TABLES = '/etc/iproute2/rt_tables' try: with open(RT_TABLES, 'a') as f: f.write('\n150 https\n') logging.info('Created new routing table for https traffic') except Exception: logging.exception('Failed to write into %s', RT_TABLES) sys.exit(1) iptables = ['iptables', '-w', '-t', 'mangle'] subprocess_call(iptables + ['-A', 'OUTPUT', '-p', 'tcp', '!', '-d', '172.16.0.0/12', '--dport', '443', '-j', 'MARK', '--set-mark', '443']) subprocess_call(iptables + ['-A', 'OUTPUT', '-p', 'udp', '--dport', '123', '-j', 'MARK', '--set-mark', '443']) subprocess_call(['ip', 'rule', 'add', 'fwmark', '443', 'lookup', 'https']) subprocess_call(['ip', 'route', 'add', 'default', 'via', nat_gateways[subnet], 'table', 'https']) # S3 is exceptional, it has it's own endpoint in VPC try: r = requests.get('https://ip-ranges.amazonaws.com/ip-ranges.json') ranges = [e['ip_prefix'] for e in r.json()['prefixes'] if e['service'] == 'S3' and e['region'] == region and 'ip_prefix' in e] except Exception: logging.exception('Failed to load ip-ranges.json') # Don't mark outgoing traffic to S3 for r in ranges: subprocess_call(iptables + ['-I', 'OUTPUT', '-d', r, '-j', 'ACCEPT'])
import sys import boto.utils import boto3 import requests import yaml from taupage import get_config FAKE_CI_ACCOUNT_KEY = "foo1234" logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) scalyr_agent_config_file = '/etc/scalyr-agent-2/agent.json' main_config = get_config() logging_config = main_config.get('logging', {}) mount_custom_log = main_config.get('mount_custom_log') rsyslog_aws_metadata = main_config.get('rsyslog_aws_metadata') application_log_path = '/var/log/application.log' custom_log_path = '/var/log-custom/*.log' auth_log_path = '/var/log/auth.log' sys_log_path = '/var/log/syslog' jwt_redaction = [{ 'match_expression': 'eyJ[a-zA-Z0-9/+_=-]{5,}\\.eyJ[a-zA-Z0-9/+_=-]{5,}\\.[a-zA-Z0-9/+_=-]{5,}', 'replacement': '+++JWT_TOKEN_REDACTED+++' }]