def __init__(self, logger=None): if logger is None: logger = CustomLogger().logger self.logger = logger self.current_path = os.path.dirname(os.path.realpath(__file__)) self.config_file_path = os.path.join(self.current_path, 'config.json') self.getconfig = self._getConfig()
def __init__(self, logger=None): self.logger = CustomLogger().logger if logger is None else logger self.certs_location = '/etc/letsencrypt/live' self.endpoint_url = os.getenv('ENDPOINT_URL') self.aws_access_key = os.getenv('AWS_ACCESS_KEY') self.aws_secret_key = os.getenv('AWS_SECRET_KEY') self.aws_region = os.getenv('AWS_REGION') self.s3_bucket_name = os.getenv('CERTS_BUCKET_NAME') self.client = self._client()
def main(): if len(sys.argv) > 2: from config import init_config init_config(sys.argv[1]) from config import init_config from config import CONFIG from logger import CustomLogger cust_logger = CustomLogger(CONFIG.web_server.logger_name) cust_logger.add_file("log/" + CONFIG.web_server.logger_name, False) import app if bool(int(sys.argv[2])) == True: app.main()
def __init__(self, provider=None): """ Automate certbot and lexicon to obtain and store let's encrypt ssl certificates into S3-compatible object storage """ self.logger = CustomLogger().logger self.dns_provider = provider self.dns_provider_username = os.getenv('DNS_PROVIDER_USERNAME') self.dns_provider_auth_token = os.getenv('DNS_PROVIDER_AUTH_TOKEN') self.client_ip_address = self._getPublicIP() self.dns_provider_update_delay = 30 self.config = Config(logger=self.logger) self.s3_store = Store(logger=self.logger) self.test = False
sys.path.append('..') sys.path.append('../../libs') import os from config import Config from twisted.internet import protocol from twisted.internet import reactor from logger import CustomLogger from mongoWebMonitor import WebServiceMonitor, StatusWebService, Logs import mongoengine import threading, time import socket import httplib import json cust_logger = CustomLogger("monitor_health_%d" % os.getpid()) config = Config() SIZE_BUFFER_HB = 21 HB_DATAGRAM = "heartbeat" TIMEOUT_HB = 10 PERIOD_CHECK_HB = 3 PERIOD_CHECK_STATUS = 5 def monitorDaemon(): pass ################ HEART BEAT MANAGEMENT ######################
# -*- coding: utf-8 -*- '''Python module for initiating and executing commands via REST API.''' # pylint: disable=too-many-branches, too-many-statements # pyling: disable=too-many-return-statements import ujson as json import requests from constants import CREDS from constants import API from logger import CustomLogger LOG = CustomLogger(__name__) class REST(object): '''Rest class for invoking REST calls GET, POST, PUT, PATCH, DELETE.''' def __init__(self, **kwargs): '''This class defines methods to invoke REST calls. Args: :pcIP (str): IP address. :username (str, optional): Username for auth. Default: 'admin'. :password (str, optional): Passwd for auth. Default: 'Password'. :port (int, optional): Port for sending REST calls. Default: 80. :baseURL (str, optional): URI for REST calls. Default: . Returns: Returns REST object instance.
import time import threading import socket from webServiceMIB import WebServiceMIB, StatusWebService from tornado.wsgi import WSGIContainer from tornado.httpserver import HTTPServer from tornado.ioloop import IOLoop ## Initializing the app app = Flask(__name__) app.debug = True config = Config() host_conf = None cust_logger = CustomLogger("web_server_%d" % os.getpid()) ## Main pages, empty for the moment @app.route('/') def index(): return render_template('index.html') #Return a fortune : line randomly selected in a file @app.route('/fortune', methods=['GET', 'POST']) def fortune(): if request.method == 'GET': cust_logger.info("Received GET request") try: file_fortune = open(
from logger import CustomLogger CUSTOM_LOGGER_HEADER = 'helper' log = CustomLogger(CUSTOM_LOGGER_HEADER).log key_set = set() try: secret_f = open('secret') line = secret_f.readline() while line: key_set.add(line) line = secret_f.readline() except FileNotFoundError: log('no secret file found, all requests will be accepted') key_set = None def is_valid_key(key): if key_set: return key and key in key_set else: return True def get_value(args, body, key, useArgs=True): argVal = args.get(key, False) bodyVal = body.get(key, False) if argVal and bodyVal: if argVal == bodyVal: return argVal
import datetime from flask import Flask, render_template, redirect, flash, url_for, request, abort, session from flask.ext.login import LoginManager, login_required, login_user, logout_user, current_user, \ current_app from flask.ext.principal import Principal, Permission, UserNeed, RoleNeed, identity_loaded,\ identity_changed, Identity import mongoengine from config import CONFIG config = CONFIG from logger import CustomLogger cust_logger = CustomLogger(config.web_server.logger_name) from models import User, Roles, LoggingHandlingException from loginForm import LoginForm, RegistrationForm app = Flask(__name__) app.config['TESTING'] = True #Secret key of the app, must be from file to prevent invalidating existing ses sions on restart app.secret_key = os.urandom(24) app.config['SESSION_COOKIE_HTTPONLY'] = True # load extension permissions principals = Principal(app) #login manager loading
""" author: arun.rs created: 26th October 2018 """ from datetime import datetime from functools import wraps from logger import CustomLogger TRACER = CustomLogger().get_logger('trace') def message(operation, type, resource, raw_resource, execution_time, status): """ :summary: Concats the supplied parameters and returns them in trace format :param operation: Operation (MySQL/ Mongo/ ES/ API/ etc) :param type: Type of the Operation (SELECT/ GET/ POST/ etc) :param resource: URL / Query / Function name :param raw_resource: URL / Query / Function name :param execution_time: Time taken to perform that operation :param status: Success or Failure :return: Concatinated string """ return "%s|%s|%s|%s|%s|%s" % (operation, type, resource, raw_resource, execution_time, status) def execution_time(start_time, end_time): """ :summary: Difference of supplied time in seconds :param start_time: Start time
section_split_model= 'section_split/models/training_unfolding_structure-2020-12-22_11-07-07_distilroberta-base' ) pgr.do_convert() document_sections = pgr.do_split() return pgr.sections_to_doccano(document_sections) return 'NOK', 400 # curl -X POST -F data='{"pilot":"Malaga","service":"Asylum Request"}' http://easyrights.linksfoundation.com/v0.3/generate @app.route('/v0.3/generate', methods=['POST']) def retrieve_pathways(): data = json.loads(request.form['data']) if data['pilot'].strip().lower() == 'malaga' and data['service'].strip( ).lower() == 'asylum request': return json.loads(open('api/malaga_pathway.json', 'r').read()) if data['pilot'].strip().lower() == 'birmingham' and data['service'].strip( ).lower() == 'clean air zone': return json.loads(open('api/birmingham_pathway.json', 'r').read()) return 'Service not available yet. Supported services: ', 400 if __name__ == '__main__': app.config['logger'] = CustomLogger('log/pgr.log') app.run(host='0.0.0.0', debug=True, port=5000)
if args.amp: print('==> Operate amp') net, optimizer = amp.initialize(net, optimizer, opt_level="O1") if args.scheduler: print('==> Operate scheduler') scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.9, patience=1, min_lr=1e-10, verbose=True) # logger here = os.getcwd() now = datetime.datetime.now() args.out = now.strftime('%Y%m%d_%H%M%S.%f') log_dir = osp.join(here, 'logs', args.out) os.makedirs(log_dir) logger = CustomLogger(out=log_dir) # make dirs for the checkpoint check_dir = osp.join(here, 'checkpoint', args.out) os.makedirs(check_dir) # for .yaml args.dataset = ['CIFAR10'] args.optimizer = 'SGD' args.model = 'ResNet18' with open(osp.join(log_dir, 'config.yaml'), 'w') as f: yaml.safe_dump(args.__dict__, f, default_flow_style=False) # Training
def init_logger(self, sess): if self.logger is None: self.logger = CustomLogger(self.config['log_dir'], sess.graph) return self.logger
import os from flask_login import LoginManager from flask import Flask, render_template from flask_sqlalchemy_session import flask_scoped_session from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from logger import CustomLogger log = CustomLogger() login_manager = LoginManager() base_dir = os.getcwd() app = Flask(__name__) login_manager.init_app(app) login_manager.login_view = 'stream' app.config.from_object(os.environ['APP_SETTINGS']) app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False engine = create_engine(os.environ['DATABASE_URL']) session_factory = sessionmaker(bind=engine) db_session = flask_scoped_session(session_factory, app) @app.route("/") def index(): return render_template("index.html")
import os import subprocess import sys import logging import StringIO import pycurl import socket import base64 try: from urllib.parse import urlencode except: from urllib import urlencode from logger import CustomLogger logger = CustomLogger(__name__).logger class CustomException(Exception): def __init__(self, code, *args): self.code = code self.msg = Error.get_code_description(code).format(*args) def __str__(self): return repr("Error: {code}: {msg}".format(code=self.code, msg=self.msg)) class Error(object): GENERIC_ERROR = 1 NOTHING_TO_DO = 2
# custom data structures from film_record import FilmRecord from logger import CustomLogger # additional modules for better UX from utils import Gauge # Global presets file_name = "movies.json" kafka_brockers = ["10.40.1.142:9092", "10.40.1.141:9092"] kafka_topic_name = "avikulin_test" kafka_clientid = "Python test util" kafka_value_serializer = FilmRecord.serialize # ! Executed code if __name__ == "__main__": logger_instance = CustomLogger("kafka_wtire_util") logger_instance.activate() films_store = list() logger_instance.get.info(f"Start reading data from file {file_name}.") with open(file_name, mode="r") as source_file: data_store = json.load(source_file) print("JSON loaded.") for i, item in enumerate(data_store): films_store.append(FilmRecord.decode(item)) print( f"Statistics: count ={len(data_store)}, collection type = {type(data_store)}" ) print(
from typing import Set, List, Tuple from kafka.structs import TopicPartition, KafkaMessage from logger import CustomLogger, LogLevels from sys import exit # Global presets kafka_brokers = ["10.40.1.142:9092", "10.40.1.141:9092"] kafka_topic_name = "avikulin_test" kafka_consumer_group_id = "test_group#111" kafka_client_id = __file__ if __name__ == "__main__": # Enable logging for kafka consumer kafka_logger = CustomLogger("kafka", log_level= LogLevels.DEBUG) kafka_logger.activate() # Enable logging for app app_logger = CustomLogger("kafka_read_util", log_level=LogLevels.INFO) app_logger.activate() while True: consumer = KafkaConsumer( kafka_topic_name, group_id=kafka_consumer_group_id, client_id=kafka_client_id, bootstrap_servers=kafka_brokers, request_timeout_ms=6001, session_timeout_ms=6000, heartbeat_interval_ms=2000,
import threading import socket import json sys.path.append('..') sys.path.append('../../libs') sys.path.append('../..') from twisted.internet import protocol from twisted.internet import reactor from twisted.internet import error from logger import CustomLogger from config import Config import random #logger for this module cust_logger = CustomLogger("loadBalancer") servers_manager = None monitors = [] config = Config() ###########SERVER MANAGER################# class ServersManager: ''' manage the different list of servers : one containing the potential servers that one can use, the other containing the servers that the load balancer is using, and the last one contains the servers that are available aka they are not working. An index number is stored to keep track of the last server used in the potential server. This is to be removed for a future use and replace by a non-linear management ''' def __init__(self, possible_servers, in_use_servers): self.available_servers = in_use_servers[:] #initialized with the in use servers
import random import time import threading import socket from twisted.internet import protocol, reactor from webServiceMIB import WebServiceMIB from logger import CustomLogger from tornado.wsgi import WSGIContainer from tornado.httpserver import HTTPServer from tornado.ioloop import IOLoop monitors = [] PERIOD_BEAT = 3 PERIOD_EXCEPTION = 2 cust_logger = CustomLogger("monitor_web_server_%d" % os.getpid()) ################# HEART BEAT MANAGEMENT ######################## #web_listen_port is the port where the webservice listen, monitor_listen_port is the listenning port of this monitor def heartbeatDaemon(web_listen_port, monitor_listen_port, list_monitors): while True: hbSocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) for monitor in list_monitors: hbSocket.sendto( "heartbeat#%d#%d" % (int(web_listen_port), int(monitor_listen_port)), (monitor['ip'], int(monitor['port_hb']))) time.sleep(PERIOD_BEAT)