def configure_logging(app): """ Sets up application wide logging. :param app: """ handler = RotatingFileHandler(app.config.get("LOG_FILE", "lemur.log"), maxBytes=10000000, backupCount=100) handler.setFormatter( Formatter("%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]")) if app.config.get("LOG_JSON", False): handler.setFormatter( logmatic.JsonFormatter(extra={"hostname": socket.gethostname()})) handler.setLevel(app.config.get("LOG_LEVEL", "DEBUG")) app.logger.setLevel(app.config.get("LOG_LEVEL", "DEBUG")) app.logger.addHandler(handler) stream_handler = StreamHandler() stream_handler.setLevel(app.config.get("LOG_LEVEL", "DEBUG")) app.logger.addHandler(stream_handler) if app.config.get("DEBUG_DUMP", False): activate_debug_dump()
def main(): # load config file global CONFIG CONFIG = json.loads(open('/root/ec2_snapshot/config.json').read()) # set up logging log_location = CONFIG['log_location'] logger = logging.getLogger() os_touch(log_location) handler = RotatingFileHandler(filename=log_location, maxBytes=8000000) handler.setFormatter(logmatic.JsonFormatter()) logger.addHandler(handler) logger.setLevel(logging.INFO) # generate snapshot_at timestamp snapshot_at = convert_dt(datetime.datetime.now()) # fire up firehose client firehose = boto3.client('firehose', 'us-west-2') # for each arn/aws account, create a thread to perform the describe-instances work threads = [] for i in range(len(CONFIG['ec2_arn'])): t = threading.Thread(target=worker(i, firehose, snapshot_at)) threads.append(t) t.start()
def initialize_logging(): # --- set up logging # remove handlers that Lambda set up, as they interfere with log configuration root_logger = logging.getLogger() if root_logger.handlers: for handler in root_logger.handlers: root_logger.removeHandler(handler) # add json log formatter handler = logging.StreamHandler() if not os.getenv("LOGFORMAT", "") == "TXT": handler.setFormatter(logmatic.JsonFormatter()) root_logger.addHandler(handler) root_logger.setLevel(os.getenv("LOGLEVEL", "DEBUG")) # don't allow boto to log at DEBUG level - it is way too chatty logging.getLogger("botocore").setLevel(logging.WARN) logging.getLogger("boto3").setLevel(logging.WARN) logging.getLogger("urllib3").setLevel(logging.WARN) LOGGER.info({ "message": "starting lambda handler {}".format(__name__), "stage": os.getenv("STAGE"), "log level": os.getenv("LOGLEVEL"), })
def loadLoggingConfig(self): fmt = "%(created)f %(msecs)d %(relativeCreated)d %(asctime)s %(levelname)s %(levelno)s %(filename)s" \ "%(args) %(funcName)s %(lineno)d %(module)s %(name)s %(pathname)s %(process)d %(processName)s " \ "%(thread)d %(threadName)s %(msg) %(message)s %(exc_info)" handler = logging.FileHandler(self.logPath) handler.setFormatter(logmatic.JsonFormatter(fmt=fmt)) self.logger.addHandler(handler)
def setup_logger(): enable_json_logger = app_config.mwdb.enable_json_logger logger = logging.getLogger("mwdb") if logger.hasHandlers(): # If already configured: return # Used by 'mwdb configure' return # Don't propagate to root logger logger.propagate = False # Setup stream handler for main logger handler = logging.StreamHandler() if enable_json_logger: formatter = logmatic.JsonFormatter( fmt="%(filename) %(funcName) %(levelname) " "%(lineno) %(module) %(threadName) %(message)" ) else: formatter = InlineFormatter( fmt="[%(levelname)s] %(threadName)s " "- %(module)s.%(funcName)s:%(lineno)s" " - %(message)s" ) handler.setFormatter(formatter) logger.addFilter(ContextFilter()) logger.addHandler(handler) logger.setLevel(logging.INFO)
def __init__(self, logger=None, date_tag=None, filehandler=None, consolehandler=None, file_id=None): if date_tag is None: date_tag = datetime.datetime.now()\ .strftime("%Y-%b-%d-%H-%M-%S") if file_id is None: # file_id = LOG_ID file_id = "test_logs" if logger is None: # logger = logging.getLogger(file_id) logger = logging.getLogger(file_id) # Add handlers and set log level if filehandler is None: logname = '-'.join([str(file_id), date_tag, '.json']) if not os.path.exists(LOG_ROOT): os.makedirs(LOG_ROOT) filehandler = logging.FileHandler(os.path.join(LOG_ROOT, logname)) filehandler.setFormatter( logmatic.JsonFormatter( extra={"hostname": socket.gethostname()})) if consolehandler is None: consolehandler = logging.StreamHandler() consolehandler.setFormatter( logmatic.JsonFormatter( extra={"hostname": socket.gethostname()})) logger.addHandler(filehandler) logger.addHandler(consolehandler) logger.setLevel(logging.DEBUG) self.logger = logger self.info = logger.info self.debug = logger.debug self.date_tag = date_tag self.filehandler = filehandler self.consolehandler = consolehandler self.file_id = file_id
def initialise_logger(target_logger, level, logging_format, reset_handlers=False): if target_logger.hasHandlers() and reset_handlers: target_logger.handlers = [] if level is not None: target_logger.setLevel(level) handler = logging.StreamHandler(sys.stdout) handler.setLevel(logging.NOTSET) target_logger.addHandler(handler) if logging_format == 'json': formatter = logmatic.JsonFormatter() handler.setFormatter(formatter)
def setup_logger(name: str, fmt: str = DEFAULT_FORMAT, level: int = DEFAULT_LEVEL): handler = logging.StreamHandler() handler.setFormatter(logmatic.JsonFormatter(fmt=fmt)) logger = logging.getLogger(name) logger.setLevel(level) logger.addHandler(handler) return logger
def define_logger(log_component='no name'): """ Define a logger with 3 default levels :param log_component: name of the component which generates the log :return: logger object """ logger = logging.getLogger(log_component) handler = logging.StreamHandler() handler.setFormatter(logmatic.JsonFormatter()) logger.addHandler(handler) logger.setLevel(logging.INFO) logger.setLevel(logging.DEBUG) return logger
def init_logging(): """ Initialize global LOGGER object with config defined in the global CONFIG object Args: None Returns: None """ if CONFIG: logging.config.dictConfig(CONFIG["logging"]) # these loggers are very noisy suppressed_loggers = [ "botocore.vendored.requests.packages.urllib3.connectionpool", "urllib3", ] for logger in suppressed_loggers: logging.getLogger(logger).setLevel(logging.ERROR) log = logging.getLogger(__name__) log.addFilter(ContextFilter()) extra = { "eventTime": datetime.datetime.now(timezone("US/Pacific")).isoformat() } log.propagate = False handler = logging.StreamHandler(sys.stdout) handler.setLevel("DEBUG") log.addHandler(handler) if CONFIG: json_logging_file = CONFIG.get("json_logging_file") if json_logging_file: if "~" in json_logging_file: json_logging_file = os.path.expanduser(json_logging_file) os.makedirs(os.path.dirname(json_logging_file), exist_ok=True) file_handler = logging.FileHandler(json_logging_file) file_handler.setFormatter(logmatic.JsonFormatter()) log.addHandler(file_handler) log = logging.LoggerAdapter(log, extra) return log
def _add_logger_handler(custom_logger, handler, formatter=None, json_formatter=False): """ Add a handler to a logger. If a formatter is not provided, a default logmatic one will be created. Args: custom_logger: The logger to add the handler to handler: The handler to add formatter: The formatter for formatting the log. See arg json_formatter for details (Optional) json_formatter: If a formatter is not provided, this identifies if the formatter that will be created should be a json formatter (logmatics) or a standard formatter """ if not formatter: if json_formatter: formatter = logmatic.JsonFormatter() else: formatter = logging.Formatter( '[%(asctime)-15s] [%(module)s] [%(funcName)s] %(levelname)s %(message)s' ) handler.setFormatter(formatter) custom_logger.addHandler(handler)
def define_logger(logger): """ """ # create a logging format as a JSON (works well for rabbit) formatter = logmatic.JsonFormatter(fmt="%(levelname) $(name) $(message)", extra={}) # Create an object which redirect logs to a text file # Keep 10 files of 5MB for history # With tuning the logs can be redirected anywhere log_dir = ROOT_DIRECTORY / 'logs' if not log_dir.exists(): log_dir.mkdir() logfilename = log_dir / 'happytal_libpython.log' handler = logging.handlers.RotatingFileHandler(str(logfilename), maxBytes=5000000, backupCount=10) handler.setLevel(logging.DEBUG) handler.setFormatter(formatter) # Multiple handlers with different types and sensitivity can be added to a unique logger logger.addHandler(handler)
def setup_logger_stdout(name, level=logging.INFO, additional_logger=[], removed_logger=[]): logger = logging.getLogger(name) handler = logging.StreamHandler(sys.stdout) # formatter = logging.Formatter() formatter = logmatic.JsonFormatter(extra={"env":os.getenv('RZC_ENV', 'local')}) handler.setFormatter(formatter) handler.setLevel(level) logger_file = logging.getLogger('logger_file') logger_file.propagate = False for n in additional_logger: logger_n = logging.getLogger(n) logger_n.addHandler(handler) logger_n.propagate = False for m in removed_logger: logger_m = logging.getLogger(m) logger_m.propagate = False logging.basicConfig(level=level, handlers=[handler]) return logger
def init_app(app): # pylint: disable=protected-access log_level = logging._nameToLevel[app.config['LOG_LEVEL']] log_type = app.config['LOG_FORMATTER'] # Configure JSON filesystem log handler handler = logging.StreamHandler(sys.stdout) if log_type.upper() == 'JSON': handler.setFormatter(logmatic.JsonFormatter()) else: #pylint: disable=line-too-long text_formatter = logging.Formatter( '%(asctime)s - %(request_id)s - %(name)s - %(filename)s - %(levelname)s - %(message)s' ) handler.setFormatter(text_formatter) handler.addFilter(RequestIdFilter()) # Configure global logging logger = logging.getLogger() logger.addHandler(handler) logger.setLevel(log_level)
def configure_logging(app): """ Sets up application wide logging. :param app: """ logfile = app.config.get("LOG_FILE", "lemur.log") # if the log file is a character special device file (ie. stdout/stderr), # file rotation will not work and must be disabled. disable_file_rotation = os.path.exists(logfile) and stat.S_ISCHR( os.stat(logfile).st_mode) if disable_file_rotation: handler = StreamHandler(open(logfile, 'a')) else: handler = RotatingFileHandler(logfile, maxBytes=10000000, backupCount=100) handler.setFormatter( Formatter("%(asctime)s %(levelname)s: %(message)s " "[in %(pathname)s:%(lineno)d]")) if app.config.get("LOG_JSON", False): handler.setFormatter( logmatic.JsonFormatter(extra={"hostname": socket.gethostname()})) handler.setLevel(app.config.get("LOG_LEVEL", "DEBUG")) app.logger.setLevel(app.config.get("LOG_LEVEL", "DEBUG")) app.logger.addHandler(handler) stream_handler = StreamHandler() stream_handler.setLevel(app.config.get("LOG_LEVEL", "DEBUG")) app.logger.addHandler(stream_handler) if app.config.get("DEBUG_DUMP", False): activate_debug_dump()
def get_logger(self, name: Optional[str] = None) -> LoggerAdapter: """Get logger.""" if self.log: return self.log if not name: name = self.get("application_name", "consoleme") level_c = self.get("logging.level", "debug") if level_c == "info": level = logging.INFO elif level_c == "critical": level = logging.CRITICAL elif level_c == "error": level = logging.ERROR elif level_c == "warning": level = logging.WARNING elif level_c == "debug": level = logging.DEBUG else: # default level = logging.DEBUG filter_c = ContextFilter() format_c = self.get( "logging.format", "%(asctime)s - %(levelname)s - %(name)s - [%(filename)s:%(lineno)s - %(funcName)s() ] - %(message)s", ) logging.basicConfig(level=level, format=format_c) logger = logging.getLogger(name) logger.addFilter(filter_c) extra = {"eventTime": datetime.datetime.now(timezone("US/Pacific")).isoformat()} now = datetime.datetime.now() # Elasticsearch logging if self.get("logging.elasticsearch_enabled", False): try: es = f"{self.get('logging.elasticsearch.host')}:{self.get('logging.elasticsearch.port')}" index_name = ( f"{self.get('logging.elasticsearch.index_name', 'consoleme')}-'" f"{now.year}{now.month}{now.day}" ) from consoleme.lib.elasticsearch import ESHandler handler = ESHandler(es, index_name) handler.setFormatter(logmatic.JsonFormatter()) handler.setLevel(self.get("logging.elasticsearch.level", "INFO")) logger.addHandler(handler) except Exception: logger.error( "Unable to configure Elasticsearch logging.", exc_info=True ) # Log to stdout and disk if self.get("logging.stdout_enabled", True): logger.propagate = False handler = logging.StreamHandler(sys.stdout) handler.setFormatter(logmatic.JsonFormatter()) handler.setLevel(self.get("logging.stdout.level", "DEBUG")) logger.addHandler(handler) logging_file = self.get("logging.file") if logging_file: if "~" in logging_file: logging_file = os.path.expanduser(logging_file) os.makedirs(os.path.dirname(logging_file), exist_ok=True) file_handler = logging.FileHandler(logging_file) file_handler.setFormatter(logmatic.JsonFormatter()) logger.addHandler(file_handler) self.log = logging.LoggerAdapter(logger, extra) return self.log
db.init_app(app) app.url_map.converters['hash64'] = HashConverter api, spec = setup_restful_service(app) logger = log.getLogger() # Don't propagate to root logger logger.propagate = False # Setup JSON stream handler for main logger handler = logging.StreamHandler() handler.setFormatter( logmatic.JsonFormatter( fmt= "%(filename) %(funcName) %(levelname) %(lineno) %(module) %(threadName) %(message)" )) logger.addHandler(handler) logger.addFilter(log.ContextFilter()) logger.setLevel(logging.INFO) @app.before_request def assign_request_id(): g.request_id = token_hex(16) g.request_start_time = datetime.utcnow() @app.after_request def log_request(response): response_time = datetime.utcnow() - g.request_start_time
def setUp(self): # access glue service self.glue = boto3.client( service_name='glue', region_name='us-east-1', endpoint_url='https://glue.us-east-1.amazonaws.com' ) # Create CloudWatch client self.cloudwatch = boto3.client('cloudwatch') # configure redshift access self.db_conn = get_redshift_connection() # access s3 storage self.s3 = boto3.resource('s3') # get json file for this test suite self.json_results = {} # define the jobs list, including initial params self.job_list = { 'EDUDirect_to_parquet_last_N_months': { 'args': { '--MONTHS': '3', '--ALL_TABLES': 'False' }, 'bucket': 'highereducation-dw-transformed-data', 'date_partition': True, 'initial_folders': [ 'EDUDirectDB' ], 'tables': [ 'cddirect_production_lead', 'cddirect_production_visitor' ], 'file_extension': 'parquet', 'job_type': ['file_creation'] }, 'EDUDirect_to_parquet_replace': { 'bucket': 'highereducation-dw-transformed-data', 'initial_folders': [ 'EDUDirectDB' ], 'tables': [ 'cddirect_production_lead_cap', 'cddirect_production_migration_versions', 'cddirect_production_school_campus_program', 'cddirect_production_school_criteria', 'cddirect_production_school_criteria_affiliate', 'cddirect_production_school_multilead_segment', 'cddirect_production_visitor_tag', 'cddirect_production_zip_state' ], 'file_extension': 'parquet', 'job_type': ['file_creation'] }, 'EDUDirect_to_parquet_new_snapshot': { 'bucket': 'highereducation-dw-transformed-data', 'initial_folders': [ 'EDUDirectDB' ], 'tables': [ 'cddirect_production_affiliate', 'cddirect_production_country', 'cddirect_production_education_level', 'cddirect_production_publisher', 'cddirect_production_school', 'cddirect_production_school_alias', 'cddirect_production_school_campus', 'cddirect_production_school_eligible_country', 'cddirect_production_school_eligible_state', 'cddirect_production_school_program', 'cddirect_production_school_program_ineligible_state', 'cddirect_production_school_provider', 'cddirect_production_school_provider_campus', 'cddirect_production_school_provider_cap', 'cddirect_production_school_provider_cap_program', 'cddirect_production_school_provider_cap_publisher', 'cddirect_production_school_provider_cap_state', 'cddirect_production_school_provider_category', 'cddirect_production_school_provider_education_level', 'cddirect_production_school_provider_leadid_flag', 'cddirect_production_school_provider_program', 'cddirect_production_school_publisher', 'cddirect_production_school_targus_score', 'cddirect_production_state', 'cddirect_production_tag', 'cddirect_production_targus_score', 'cddirect_production_user', 'cddirect_production_widget_category', 'cddirect_production_widget_degree', 'cddirect_production_widget_degree_recommendation', 'cddirect_production_widget_subject', 'cddirect_production_widget_subject_alias', 'cddirect_production_widget_subject_recommendation', 'form_position_csv' ], 'date_partition': True, 'file_extension': 'parquet', 'job_type': ['file_creation'] }, 'EDUDirect_to_parquet_current_dimensions': { 'bucket': 'highereducation-dw-transformed-data', 'initial_folders': ['EDUDirectDB-current'], 'files': [ 'cddirect_production_affiliate', 'cddirect_production_country', 'cddirect_production_lead_cap', 'cddirect_production_publisher', 'cddirect_production_school', 'cddirect_production_school_program', 'cddirect_production_school_provider', 'cddirect_production_school_provider_cap', 'cddirect_production_school_provider_category', 'cddirect_production_school_provider_program', 'cddirect_production_school_provider_education_level', 'cddirect_production_state', 'cddirect_production_user', 'cddirect_production_widget_category', 'cddirect_production_widget_degree', 'cddirect_production_widget_subject', 'cddirect_production_widget_degree_recommendation', 'cddirect_production_widget_subject_recommendation', 'form_position_csv', ], 'file_extension': 'parquet', 'job_type': ['file_creation'] }, 'EDUDirect_user_agent': { 'args': { '--TYPE': 'historical', }, 'bucket': 'highereducation-dw-transformed-data', 'tables': ['user_agent'], 'file_extension': 'parquet', 'job_type': ['file_creation'] }, 'EDUDirect_to_staging': { 'args': { '--TYPE': 'historical', '--ENVIRONMENT': 'dev', '--START_DATE': '000', '--END_DATE': '000', }, 'bucket': 'highereducation-dw-staging-data', 'initial_folders': ['EDUDirectDB', 'tmp'], 'date_partition': True, 'file_extension': 'parquet', 'job_type': ['file_creation'] }, 'EDUDirect_related_subject': { 'args': { '--TYPE': 'historical', '--ENVIRONMENT': 'dev', '--DATABASE': 'highereducation-dw-edudirectdb-parquet-current', '--START_DATE': '000', '--END_DATE': '000', }, 'bucket': 'highereducation-dw-staging-data', 'initial_folders': ['EDUDirectDB', 'env'], 'date_partition': True, 'file_extension': 'parquet', 'job_type': ['file_creation'] }, # migration to redshift 'UpdateCrawler': { 'args': { '--CRAWLER': 'highereducation-dw-edudirectdb-staging' }, 'job_type': ['update_db'] }, 'Migrate_EDUDirect_to_Redshift': { 'args': { '--ORIGIN': 'stag-platformevents-db-staging.normalized_events_fact_table_stag_current', #database and table to query '--DESTINATION': 'stag_platform_events_staging_internal.normalized_events_fact_table_stag_current', }, 'job_type': ['update_db'], 'output_db': 'stag_platform_events_staging_internal', 'output_table': 'normalized_events_fact_table_stag_current' ' }, } # initialize logger self.logger = logging.getLogger() handler = logging.StreamHandler() handler.setFormatter(logmatic.JsonFormatter()) self.logger.addHandler(handler) self.logger.setLevel(logging.INFO)
import logging import logging.handlers import sys import socket import logmatic logger = logging.getLogger() sh = logging.handlers.SysLogHandler('/dev/log') sh.setFormatter(logmatic.JsonFormatter(extra={"hello": "world","hostname":socket.gethostname()},prefix="appname: ")) logger.addHandler(sh) logger.setLevel(logging.INFO) test_logger = logging.getLogger("test") test_logger.info({"special": "value", "run": 12}) test_logger.info("classic message", extra={"special": "value", "run": 12}) def exception_test(): try: raise Exception('test') except Exception: test_logger.exception("This is a fake exception") exception_test()
#!/usr/bin/env python import logmatic import logging import socket """ https://github.com/logmatic/logmatic-python """ logger = logging.getLogger() handler = logging.StreamHandler() handler.setFormatter( logmatic.JsonFormatter(extra={"hostname": socket.gethostname()})) logger.addHandler(handler) logger.setLevel(logging.INFO) test_logger = logging.getLogger("test") test_logger.info("classic message", extra={"special": "value", "run": 12}) logger.warn("No user currently authenticated.", extra={ "customer": "my_beloved_customer", "login_name": "*****@*****.**" })
import logging import sys import socket sys.path.append('logmatic/') import logmatic logger = logging.getLogger() handler = logging.StreamHandler() handler.setFormatter(logmatic.JsonFormatter(extra={"hello": "world","hostname":socket.gethostname()})) logger.addHandler(handler) logger.setLevel(logging.INFO) test_logger = logging.getLogger("test") test_logger.info({"special": "value", "run": 12}) test_logger.info("classic message", extra={"special": "value", "run": 12}) def exception_test(): try: raise Exception('test') except Exception: test_logger.exception("This is a fake exception") exception_test()
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = get_key() alert = False path = os.path.expanduser('~/python-logs') logfile = os.path.expanduser('~/python-logs/security.log') if os.path.isdir(path): pass else: os.mkdir(path) logger = logging.getLogger("Rotating Log") logger.setLevel(logging.INFO) handler = RotatingFileHandler(logfile, maxBytes=5 * 1024 * 1024, backupCount=5) handler.setFormatter(logmatic.JsonFormatter()) logger.addHandler(handler) for project in get_projects(): try: service = discovery.build('compute', 'v1') request = service.networks().list(project=project) response = request.execute() items = response['items'] for item in items: vpc = item['name'] autocreate = item['autoCreateSubnetworks'] if vpc == 'default' and autocreate is True: alert = True
parser.set_defaults(attrs=[]) parser.set_defaults(debug=False) parser.set_defaults(docker_version="auto") parser.set_defaults(skip_name=None) parser.set_defaults(skip_image=None) parser.set_defaults(match_name=None) parser.set_defaults(match_image=None) parser.set_defaults(match_label=None) parser.set_defaults(timeout=120) args = parser.parse_args() # Initialise the logger for Logmatic.io logmatic_logger = logging.getLogger("docker-logmatic") handler = logmatic.LogmaticHandler(args.token, host=args.hostname, port=args.port, ssl=args.ssl) handler.setFormatter(logmatic.JsonFormatter(fmt="%(message)")) logmatic_logger.addHandler(handler) logmatic_logger.setLevel(logging.DEBUG) logmatic_logger.propagate = False if args.debug is True: internal_logger.setLevel(logging.DEBUG) sys_handler = logging.StreamHandler(sys.stderr) internal_logger.addHandler(sys_handler) internal_logger.debug(args) else: internal_logger.disabled = True # Initialise the connection to the local daemon base_url = 'unix://var/run/docker.sock'
def setUp(self): # access glue service self.glue = boto3.client( service_name='glue', region_name='us-east-1', endpoint_url='https://glue.us-east-1.amazonaws.com' ) # Create CloudWatch client self.cloudwatch = boto3.client('cloudwatch') # access s3 storage self.s3 = boto3.resource('s3') # get json file for this test suite self.json_results = {} # define the jobs list, including initial params self.job_list = { 'EDUDirect_to_parquet_current_day': { 'bucket' : 'highereducation-dw-transformed-data', 'initial_folders': ['EDUDirectDB-current'], 'tables': [ 'cddirect_production_lead', 'cddirect_production_visitor' ], 'file_extension': 'parquet' }, 'EDUDirect_user_agent': { 'args': { '--TYPE': 'current_day' }, 'bucket' : 'highereducation-dw-transformed-data', 'initial_folders': [ 'EDUDirectDB-current' ], 'tables': [ 'user_agent' ], 'file_extension': 'parquet' }, 'EDUDirect_to_staging': { 'args': { '--TYPE': 'current_day', '--ENVIRONMENT': 'dev', '--START_DATE': '000', '--END_DATE': '000', }, 'bucket': 'highereducation-dw-staging-data', 'inital_folders': ['EDUDirectDB','tmp'], 'tables': [ 'lead_fact_table_dev_current_v1' ], 'date_partition': True, 'file_extension': 'parquet' }, 'EDUDirect_related_subject': { 'args': { '--TYPE': 'current_day', '--DATABASE': 'highereducation-dw-edudirectdb-parquet-current', '--ENVIRONMENT': 'dev', '--START_DATE': '000', '--END_DATE': '000', }, 'bucket': 'highereducation-dw-staging-data', 'inital_folders': ['EDUDirectDB','dev'], 'tables': [ 'lead_fact_table_env_current' ], 'date_partition': True }, } # initialize logger self.logger = logging.getLogger() handler = logging.StreamHandler() handler.setFormatter(logmatic.JsonFormatter()) self.logger.addHandler(handler) self.logger.setLevel(logging.INFO)
unk_vnf_coll = os.environ['UNK_COLL'] log_level = os.environ['LOG_LEVEL'] # cat_url = "http://pre-int-sp-ath.5gtango.eu:4011/catalogues/api/v2/" # db_host = "mongo" # db_port = 27017 # db_name = "tng-sdk-analyze-weight" # dict_coll = "dictionaries" # unk_vnf_coll = "unknown_vnfs" # enc_fig_coll = "encoded_figs" # log_level = "INFO" logger = logging.getLogger() handler = logging.StreamHandler() handler.setFormatter( logmatic.JsonFormatter(extra={"hostname": "tng-sdk-analyze-weight"})) logger.addHandler(handler) level = logging.getLevelName(log_level) enc_fig_coll = "encoded_figs" logger.setLevel(level) # Create a URL route in our application for "/" @app.route('/tng-sdk-analyze-weight/api/weight/v1/home') def home(): logger.info("Logging home end point") return render_template('index.html') @app.route('/tng-sdk-analyze-weight/api/weight/v1/mgmt') def mgmt():
def setUp(self): # access glue service self.glue = boto3.client( service_name='glue', region_name='us-east-1', endpoint_url='https://glue.us-east-1.amazonaws.com') # Create CloudWatch client self.cloudwatch = boto3.client('cloudwatch') # access s3 storage self.s3 = boto3.resource('s3') # get json file for this test suite self.json_results = {} # define the jobs list, including initial params self.job_list = { 'PlatformEvents_cap_info_to_parquet': { 'args': { '--TYPE': 'current_day' }, 'bucket': 'highereducation-dw-transformed-data', 'initial_folders': ['PlatformEvents-current'], 'tables': ['cap_info_csv'], 'file_extension': 'parquet' }, 'PlatformEvents_to_parquet': { 'args': { '--MONTHS': '3', }, 'bucket': 'highereducation-dw-transformed-data', 'initial_folders': ['PlatformEvents'], 'tables': ['platform_events_public_normalized_events'], 'date_partition': True, 'file_extension': 'parquet' }, 'PlatformEvents_to_staging': { 'args': { '--TYPE': 'current_day', '--ENVIRONMENT': 'dev' }, 'bucket': 'highereducation-dw-staging-data', 'initial_folders': ['PlatformEvents', 'tmp'], 'tables': ['fact_table_dev_current_v1'], 'date_partition': True, 'file_extension': 'parquet' }, 'PlatformEvents_prices': { 'args': { '--TYPE': 'current_day', '--ENVIRONMENT': 'dev' }, 'bucket': 'highereducation-dw-staging-data', 'initial_folders': ['PlatformEvents', 'dev'], 'tables': ['normalized_events_fact_table_dev_current'], 'date_partition': True, 'file_extension': 'parquet' }, } # initialize logger self.logger = logging.getLogger() handler = logging.StreamHandler() handler.setFormatter(logmatic.JsonFormatter()) self.logger.addHandler(handler) self.logger.setLevel(logging.INFO)