def __init__(self, path: str, name: str, whitelist_path: str, verify_vars: bool, verify_secrets: bool, upload_vars_from_file: str, verify_vars_from_files: list, proxy_node=False): self.path: str = path # sesam-master-node-config/ self.name: str = name # master||extra self.node_path = path + '/node' self.whitelist_path: str = self.node_path + '/' + whitelist_path self.verify_vars: bool = verify_vars self.verify_secrets: bool = verify_secrets self.proxy_node = proxy_node self.read_variables_file = None self.upload_vars_from_file = None if upload_vars_from_file is not None: self.read_variables_file = True self.upload_vars_from_file: str = self.node_path + '/' + upload_vars_from_file self.verify_vars_from_files: list = [ self.node_path + '/' + f for f in verify_vars_from_files ] self.pipes = {} self.conf = [] self.config_vars = [] self.config_secrets = [] self.upload_vars = {} self.upload_secrets = {} self.LOGGER = sesam_logger(f'Node {self.name}')
def elwin_workorder(s: Session, url: str, entity: dict): logger = sesam_logger("elwin_workorder_handler") def do_request(session: Session, method: str, entity: dict): resp = None if method == 'update': resp = session.put(f'{url}/ElWinAvtaler/api/workorders', json=entity) elif method == 'get': resp = session.get( f'{url}/ElWinAvtaler/api/workorders?externalIds={entity["ExternalId"]}' ) elif method == 'create': resp = session.post(f'{url}/ElWinAvtaler/api/workorders', json=entity) else: logger.critical( f'Critical internal error. "{method}" not valid! Please verify the code. Exiting' ) exit(-1) returnval = resp.content.decode('UTF-8') logger.debug( f'Method "{method}" for "{entity["_id"]}" gave "{returnval}" & status code: "{resp.status_code}"' ) return returnval logger.debug(f'Proccessing entity {entity.get("_id", None)}') # If entity has Id then we just update if entity.get('Id', None) is not None: do_request(s, 'update', entity) else: # Try to find entity based on externalId # If response is not JSON then we need to create the entity try: if "ExternalId" in entity: response_entity = json.loads(do_request(s, 'get', entity)) else: response_entity = {} if type(response_entity) == list: if len(response_entity) == 1: response_entity = response_entity[0] else: response_entity = {} else: response_entity = {} except json.JSONDecodeError as e: logger.debug( f'Could not GET entity with externalId: "{entity.get("ExternalId", None)}" because of error: "{e}"' ) response_entity = {} # If we find the Id then we update, else we create. if response_entity.get('Id', None) is not None: entity['Id'] = response_entity['Id'] do_request(s, 'update', entity) else: if 'Id' in entity: del entity['Id'] do_request(s, 'create', entity)
def __init__(self, url, git_token, mount_point): self.LOGGER = sesam_logger('KeyVault') self.client = Client(url=url) self.mount_point = mount_point self.client.auth.github.login(git_token) self.missing_secrets = [] if not self.client.is_authenticated(): self.LOGGER.critical(f'Cannot authenticate vault {url}. Exiting.') exit(-1)
def __init__(self, url, username, password_or_token, folder, branch): self.url = url self.username = username self.password_or_token = password_or_token self.folder = folder self.branch = branch self.LOGGER = sesam_logger('Git') self.repo = self.clone_repo()
def test_sesam_logger(): with LogCapture() as expected_logger: actual_logger = sesam_logger('test_sesam_logger') actual_logger.setLevel(level='DEBUG') actual_logger.info('a info') actual_logger.error('an error') actual_name = actual_logger.name expected_logger.check( (actual_name, 'WARNING', 'Unsupported value or no LOG_LEVEL provided. ' 'Hence, setting default log level to INFO.'), (actual_name, 'INFO', 'a info'), (actual_name, 'ERROR', 'an error'), )
def elwin_workorder(s: Session, url: str, entity: dict): logger = sesam_logger("elwin_workorder_handler") def do_workorders_request(session: Session, method: str, entity: dict): resp = None #if method == 'update': # We won't update # resp = session.put(f'{url}/ElWinAvtaler/api/workorders', json=entity) if method == 'get': resp = session.get(f'{url}/ElWinAvtaler/api/workorders?externalIds={entity["ExternalId"]}') elif method == 'create': resp = session.post(f'{url}/ElWinAvtaler/api/workorders', json=entity) else: logger.critical(f'Critical internal error. "{method}" not valid! Please verify the code. Exiting') exit(-1) returnval = resp.content.decode('UTF-8') logger.debug(f'Method "{method}" for "{entity["_id"]}" gave "{returnval}" & status code: "{resp.status_code}"') return returnval def do_workordermessages_request(session: Session, entity: dict): resp = session.post(f'{url}/ElWinAvtaler/api/workordermessages', json=entity) returnval = resp.content.decode('UTF-8') logger.debug(f'POST to workordermessages gave "{returnval}" & status code: "{resp.status_code}"') return returnval logger.debug(f'Proccessing entity {entity.get("_id", None)}') # If entity has Id then we just update message_entity = { "WorkorderId": 0, #response_entity.get('Id'), "Message": entity.get('Description') # response_entity.get('Description') } if entity.get('Id', None) is not None: logger.warning('We wont update this entity.') pass #We wont update workorders. # try: # do_workorders_request(s, 'update', entity) # message_entity['WorkorderId'] = entity.get('Id', 0) # except json.JSONDecodeError as e: # logger.debug(f'Could not UPDATE entity with id: "{entity.get("_id", None)}" because of error: "{e}"') else: # Try to find entity based on externalId # If response is not JSON then we need to create the entity try: if "ExternalId" in entity: response_entity = json.loads(do_workorders_request(s, 'get', entity)) else: response_entity = {} if type(response_entity) == list: if len(response_entity) == 1: response_entity = response_entity[0] else: response_entity = {} else: response_entity = {} except json.JSONDecodeError as e: logger.debug( f'Could not GET entity with externalId: "{entity.get("ExternalId", None)}" because of error: "{e}". Entity: {entity}') response_entity = {} # If we find the Id then we update, else we create. if response_entity.get('Id', None) is not None: logger.warning('We wont update this entity.') pass # We don't update. # entity['Id'] = response_entity['Id'] # try: # do_workorders_request(s, 'update', entity) # message_entity['WorkorderId'] = response_entity.get('Id', 0) # except json.JSONDecodeError as e: # logger.debug(f'Could not UPDATE entity with id: "{entity.get("_id", None)}" because of error: "{e}". Entity: {entity}') else: if 'Id' in entity: del entity['Id'] try: created_entity = json.loads(do_workorders_request(s, 'create', entity)) message_entity['WorkorderId'] = created_entity.get('Id', 0) if message_entity.get('WorkorderId', 0) != 0 or message_entity.get('Description') is not None: do_workordermessages_request(s, message_entity) else: logger.error( f'Could not send workordermessage request because the description or workorderid is null: Entity: {message_entity}') except json.JSONDecodeError as e: logger.debug(f'Could not CREATE entity with id: "{entity.get("_id", None)}" because of error: "{e}". Entity: {entity}')
def test_logger_with_timestamp(): logger = sesam_logger('test_logger_with_timestamp', timestamp=True) assert logger.handlers[ 0].formatter._fmt == '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
from json import loads as load_json, dumps as dump_json from os import listdir from re import sub from Node import Node from dotty_dict import dotty from sesamutils import sesam_logger LOGGER = sesam_logger('config-creator') class ConfigTemplates: def __init__(self, path): self.pipe_on_extra_from_extra_to_master = None self.pipe_on_extra_from_master_to_extra = None self.system_on_extra_from_extra_to_master = None self.system_on_extra_from_master_to_extra = None self.pipe_on_master_from_extra_to_master = None self.pipe_on_master_from_master_to_extra = None self.system_on_master_from_extra_to_master = None self.system_on_master_from_master_to_extra = None self.node_metadata = None self.get_templates(path) def get_templates(self, path): for f in listdir(path): if f == 'node-metadata.conf.json': self.node_metadata = load_json(open(f'{path}{f}', 'r').read())
from datetime import datetime import json from flask import Flask, request, jsonify, Response, abort import requests from requests.exceptions import Timeout from sesamutils import sesam_logger, VariablesConfig from sesamutils.flask import serve from collections import OrderedDict app = Flask(__name__) logger = sesam_logger('zendesk', app=app, timestamp=True) # Default values can be given to optional environment variables by the use of tuples required_env_vars = ["USER", "TOKEN","SUBDOMAIN"] optional_env_vars = [("DEBUG","false"),("LOG_LEVEL", "INFO"),"DUMMY_URL"] config = VariablesConfig(required_env_vars, optional_env_vars=optional_env_vars) if not config.validate(): logger.error("Environment variables do not validate. Exiting system.") os.sys.exit(1) USER = config.USER TOKEN = config.TOKEN ZEN_AUTH = (USER+'/token', TOKEN) SUBDOMAIN = config.SUBDOMAIN DUMMY_URL = False
from slack.errors import SlackApiError # Local imports from Node import Node from Vaulter import Vaulter from config_creator import generate_config, get_vars_from_master from gitter import Gitter class AppConfig(object): pass config = AppConfig() LOGGER = sesam_logger('Autodeployer') LOGGER.debug(listdir()) ENV_VARS = [('NODE_FOLDER', str, None), ('ENVIRONMENT', str, None), ('VERIFY_SECRETS', bool, [('VAULT_GIT_TOKEN', str, None), ('VAULT_MOUNTING_POINT', str, None), ('VAULT_URL', str, None)]), ('VERIFY_VARIABLES', bool, None), ('MASTER_NODE', dict, { "URL": str, "JWT": str, "UPLOAD_VARIABLES": bool, "UPLOAD_SECRETS": bool, "CONFIG_GROUP": str }), ('EXTRA_NODES', dict, None), ('DRY_RUN', bool, None), ('SLACK_API_TOKEN', str, None), ('SLACK_CHANNEL', str, None), ('RELEASE_URL', str, None)]
import os import sys import json import tempfile import socket from flask import Flask, abort, send_file, jsonify from smb.SMBConnection import SMBConnection from sesamutils import VariablesConfig, sesam_logger from sesamutils.flask import serve from validator import validate_file APP = Flask(__name__) logger = sesam_logger("cifs-reader", app=APP) required_env_vars = ["username", "password", "hostname", "host", "share"] optional_env_vars = ["schema_path"] config = VariablesConfig(required_env_vars, optional_env_vars) if not config.validate(): sys.exit(1) def create_connection(): return SMBConnection(config.username, config.password, socket.gethostname(), config.hostname, is_direct_tcp=True, use_ntlm_v2=True)
app = Flask(__name__) # set GOOGLE_APPLICATION_CREDENTIALS envvar for GCP authentication os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'service_account_key.json' required_env_vars = [ "GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_APPLICATION_CREDENTIALS_CONTENT", "QUERY_CONFIGS" ] optional_env_vars = [("DEFAULT_PAGE_SIZE", 100)] env_config = VariablesConfig(required_env_vars, optional_env_vars) if not env_config.validate(): sys.exit(1) logger = sesam_logger('google-bigquery', app=app) env_config.QUERY_CONFIGS = json.loads(env_config.QUERY_CONFIGS) env_config.DEFAULT_PAGE_SIZE = int(env_config.DEFAULT_PAGE_SIZE) logger.info('started up with\n\tQUERY_CONFIGS:{}'.format( env_config.QUERY_CONFIGS)) # write out service config from env var to known file with open(env_config.GOOGLE_APPLICATION_CREDENTIALS, "wb") as out_file: out_file.write(env_config.GOOGLE_APPLICATION_CREDENTIALS_CONTENT.encode()) def stream_rows(query_key, since, limit, page_size): is_first_yield = True query_config = env_config.QUERY_CONFIGS.get(query_key)
from sesamutils import sesam_logger, VariablesConfig import os import requests import json # Import smtplib for the actual sending function import smtplib # Import the email modules we'll need from email.message import EmailMessage from collections import OrderedDict app = Flask(__name__) logger = sesam_logger('smtp-bundle', app=app, timestamp=True) required_env_vars = [ "MAIL_SERVER", "SENDER", "RCPT", "MAIL_PASSWORD", "MAIL_USERNAME", "SESAM_JWT", "BASE_URL" ] optional_env_vars = [("MAIL_PORT", 587), ("LOG_LEVEL", "info"), ("MAIL_USE_TLS", True), ("MAIL_USE_SSL", False), ("BUNDLE_SIZE", 2), ("SEND_UNTIL_FIXED", False)] config = VariablesConfig(required_env_vars, optional_env_vars=optional_env_vars) mail = Mail(app) def stream_json(entities):
from requests import Session from sesamutils import sesam_logger from sesamutils.flask import serve from flask import Flask, abort, Response import time app = Flask(__name__) logger = sesam_logger("dsb-proxy", app=app) now = time.time() URL = f"https://innmelding.dsb.no/elvirksomhetsregisteret/virksomhetssok?0-1.0-searchForm=&antiCache={now}" @app.route("/", methods=['GET']) def process_request(): logger.info(f"processing request for {URL}") session = Session() session.get(URL) resp = session.get(URL) try: resp.raise_for_status() except Exception as e: logger.error(f"Failed to get spreadsheet from host:\n{e}") abort(500, e) content_disposition = resp.headers.get("Content-Disposition")
import os from flask import Flask, request, jsonify, Response, abort from sesamutils import sesam_logger, VariablesConfig from sesamutils.flask import serve required_env_vars = ["SUBDOMAIN"] optional_env_vars = [ "DEBUG", "LOG_LEVEL", ("API_ROOT", "zendesk.com/api/v2/tickets/") ] # Default values can be given to optional environment variables by the use of tuples app = Flask(__name__) logger = sesam_logger('DemoMicroservice', app=app, timestamp=True) orders = [{ 'id': 1, 'Username': u'Unjudosely', 'Orders': u'Thinkpad', 'TotalSum': 8000 }, { 'id': 2, 'Username': u'Wimen1979', 'Orders': u'MacbookPro', 'TotalSum': 12000 }, { 'id': 3, 'Username': u'Gotin1984', 'Orders': u'Chormebook', 'TotalSum': 10000
from flask import Flask, request, Response, abort import requests import zlib from sesamutils import sesam_logger from sesamutils.flask import serve app = Flask(__name__) logger = sesam_logger("compression-service", app=app) def decode_file(url): d = zlib.decompressobj(16 + zlib.MAX_WBITS) with requests.get(url, stream=True) as r: for chunk in r.iter_content(chunk_size=1048576, decode_unicode=False): if chunk: yield d.decompress(chunk) @app.route('/gzip', methods=['GET']) def get_gzip(): full_url = request.args.get('url', None) # use default value replace 'None' logger.info(f"processing request for {full_url}") if not full_url: return abort(400, "Missing url param in request.") return Response(decode_file(full_url), mimetype='application/json', direct_passthrough=True)
import os import json from sesamutils import sesam_logger, Dotdictify import schedule import time # fetch env vars jwt = os.environ.get('JWT') node_api = os.environ.get('NODE_API') node_subscription = os.environ.get('NODE_SUBSCRIPTION') portal_api = os.environ.get('PORTAL_API') schedule_interval = int(os.environ.get('SCHEDULE_INTERVAL', 60)) # default 60 seconds # set logging logger = sesam_logger('sesam-role-handler', timestamp=True) # API urls available_roles_api = portal_api + '/subscriptions/' + node_subscription + '/available-roles' pipes_api = node_api + '/pipes' systems_api = node_api + '/systems' member_roles_api = portal_api + '/subscriptions/' + node_subscription + '/members/' permissions_api = node_api + '/permissions/' headers = { 'Content-Type': 'application/json', 'Authorization': 'Bearer ' + jwt } def get_available_roles():
from flask import Flask, request, abort import requests import uuid from sesamutils.flask import serve from sesamutils import sesam_logger, VariablesConfig app = Flask(__name__) logger = sesam_logger("Slack command receiver",) req_vars = ["NODE_JWT", "VALIDATION_TOKEN", "ENDPOINT_URL"] config = VariablesConfig(req_vars) if not config.validate(): logger.error("Missing required variable(s).") headers = { "Authorization": f"bearer {config.NODE_JWT}" } def valid_token(token): return token == config.VALIDATION_TOKEN @app.route("/command", methods=["POST"]) def command(): _id = str(uuid.uuid1()) data = dict(request.form) data["_id"] = _id
from functools import wraps from flask import Flask, request, Response, abort import json from filehandling import stream_file_by_row, stream_file_by_col from sesamutils import sesam_logger import threading import os app = Flask(__name__) logger = sesam_logger("datasource-service", app=app) _lock_config = threading.Lock() requiredVars = ['file_url'] optionalVars = ['sheet', 'start', 'direction', 'names', 'ids', 'since'] logger.info(f"------Starting MS------- {requiredVars}") ## Helper functions # def check_env_variables(required_env_vars, missing_env_vars): # for env_var in required_env_vars: # value = os.getenv(env_var) # if not value: # missing_env_vars.append(env_var) # # if len(missing_env_vars) != 0: # app.logger.error(f"Missing the following required environment variable(s) {missing_env_vars}") # sys.exit(1)
from requests_oauthlib import OAuth2Session import requests import datetime import json import logging import os import re import sys import urllib.parse from sesamutils import sesam_logger, Dotdictify from sesamutils.flask import serve app = Flask(__name__) logger = sesam_logger("incremental-jsonsystem", app=app) SYSTEM = None FULL_URL_PATTERN = None UPDATED_URL_PATTERN = None UPDATED_PROPERTY = None OFFSET_BIGGER_AND_EQUAL = None def get_var(var): envvar = None envvar = os.getenv(var.upper()) logger.debug("Setting %s = %s" % (var, envvar)) return envvar
from flask import Flask, request, jsonify, Response import json import requests import os import sys from sesamutils import VariablesConfig, sesam_logger app = Flask(__name__) logger = sesam_logger("Steve the logger", app=app) ## Logic for running program in dev try: with open("helpers.json", "r") as stream: logger.info("Setting env vars via helpers.json") env_vars = json.load(stream) os.environ['github_username'] = env_vars['github-username'] os.environ['github_token'] = env_vars['github-token'] os.environ['github_base_url'] = env_vars['github-base-url'] except OSError as e: logger.info("Using env vars defined in SESAM") ## Helpers required_env_vars = ['github_username', 'github_token', 'github_base_url'] optional_env_vars = ["page_size"] username = os.getenv('github_username') token = os.getenv('github_token') @app.route('/') def index(): output = {
""" import os import tempfile import logging import json import requests from flask import Flask, Response, request from sesamutils import VariablesConfig, sesam_logger from sesamutils.flask import serve from str_utils import str_to_bool APP = Flask(__name__) logger = sesam_logger("file-transfer-service", app=APP) required_env_vars = ["UPLOAD_URL"] optional_env_vars = [("FILE_URL", "file_url"), ("FILE_NAME", "file_id"), ("TARGET_PATH", "local_path"), ("TARGET_PATH_IN_URL", "false"), ("CHUNK_SIZE", 262144 * 4 * 10), # chunk size 10Mb ("FAIL_FAST_ON_ERROR", "false")] config = VariablesConfig(required_env_vars, optional_env_vars) if not config.validate(): exit(1) @APP.route("/transfer", methods=['POST'])
import logging from sesamutils import VariablesConfig, sesam_logger from graph import Graph from utils import stream_json, determine_url_parts app = Flask(__name__) # Environment variables required_env_vars = [ "client_id", "client_secret", "grant_type", "resource", "entities_path", "next_page", "token_url" ] optional_env_vars = ["log_level", "base_url", "sleep", "sharepoint_url"] logger = sesam_logger("o365graph") config = VariablesConfig(required_env_vars, optional_env_vars=optional_env_vars) if not config.validate(): sys.exit(1) data_access_layer = Graph(config) @app.route("/entities/<path:path>", methods=["GET", "POST"]) def get(path): if request.method == "POST": path = request.get_json() if request.method == "GET":
#!/usr/bin/env python3 from flask import Flask, Response, request, abort from sesamutils.flask import serve from sesamutils import sesam_logger from sesamutils import VariablesConfig import json import requests import sys import time # Activate logging logger = sesam_logger("sap-odata-source") # Get env.vars required_env_vars = ["SERVICE_URL"] optional_env_vars = [ "LOG_LEVEL", ("AUTH_TYPE", "basic"), "USERNAME", "PASSWORD", "TOKEN_URL", "TOKEN_REQUEST_HEADERS", "TOKEN_REQUEST_BODY" ] env_vars = VariablesConfig(required_env_vars, optional_env_vars=optional_env_vars) # Check that all required env.vars are supplied if not env_vars.validate(): sys.exit(1) # Verify authentication supported_auth_types = ["basic", "token"] if env_vars.AUTH_TYPE.lower() not in supported_auth_types:
from requests_oauthlib import OAuth2Session import json import os import copy import requests import datetime from jinja2 import Template, Environment, meta from sesamutils import sesam_logger from sesamutils.flask import serve import re app = Flask(__name__) PORT = int(os.environ.get("PORT", 5001)) logger = sesam_logger("rest-transform-service", app=app) prop = os.environ.get("PROPERTY", "response") payload_property = os.environ.get("PAYLOAD_PROPERTY_FOR_TRANSFORM_REQUEST", "payload") method = os.environ.get("METHOD", "GET").upper() url = os.environ["URL"] headers = json.loads(os.environ.get("HEADERS", "{}")) authorization = os.environ.get("AUTHORIZATION") do_stream = os.environ.get("DO_STREAM", "true").lower() == "true" do_verify_ssl = os.environ.get("DO_VERIFY_SSL", "false").lower() == "true" tolerable_status_codes = os.environ.get("TOLERABLE_STATUS_CODES") service_config_property = os.environ.get("SERVICE_CONFIG_PROPERTY", "service_config") print(f"starting with {url}, do_stream={do_stream}, prop={prop}, tolerable_status_codes='{tolerable_status_codes}'") session_factory = None
def test_if_logger_name_is_blank(): with raises(ValueError): logger = sesam_logger('')
from flask import Flask, request, Response, abort from werkzeug.exceptions import BadRequestKeyError import requests from azure.servicebus import Message, TopicClient, SubscriptionClient, QueueClient from azure.servicebus.common.errors import NoActiveSession from azure.servicebus.common.constants import NEXT_AVAILABLE import json import os import sys from sesamutils import sesam_logger from sesamutils.flask import serve app = Flask(__name__) logger = sesam_logger("azure-service-bus", app) try: PORT = int(os.environ.get('PORT', 5000)) CONNECTION_LIST = json.loads(os.environ['CONNECTION_LIST']) DEFAULT_IDLE_TIMEOUT = int(os.environ.get('DEFAULT_IDLE_TIMEOUT', 30)) DEFAULT_PREFETCH = int(os.environ.get('DEFAULT_PREFETCH', 30)) except (ValueError, KeyError) as er: logger.exception(er) sys.exit(1) def respond(status_code, message): return Response(response=json.dumps({ "success": str(status_code)[0] in ['2', '3'], 'message':
def test_if_logger_name_is_none(): with raises(ValueError): logger = sesam_logger(None)
def test_logger_name_is_set(): logger = sesam_logger('test_logger_name_is_set') assert logger.name == "test_logger_name_is_set"
""" from requests import session from sesamutils import VariablesConfig, sesam_logger required_env_vars = ["INPUT_URL", "OUTPUT_URL"] optional_env_vars = [("INPUT_JWT", None), ("OUTPUT_JWT", None), ("OUTPUT_CONTENT_TYPE", "application/json; charset=utf-8"), ("LOG_TIMESTAMP", True)] config = VariablesConfig(required_env_vars, optional_env_vars) if not config.validate(): exit(1) logger = sesam_logger("file-transfer-service", timestamp=config.LOG_TIMESTAMP) def main(): try: input_connection = session() try: input_connection.headers[ 'Authorization'] = f'bearer {config.INPUT_JWT}' except AttributeError: pass logger.debug(f'Creating stream from input URL "{config.INPUT_URL}"') res = input_connection.get(config.INPUT_URL) res.raise_for_status()