def test_default_logger_console_formatter_parameters(): """Test default logger level stream formatter parameters""" main_log = simplelogging.get_logger("__main__") log_colors = { "DEBUG": "blue", "INFO": "black,bg_green", "WARNING": "black,bg_yellow", "ERROR": "white,bg_red", "CRITICAL": "red,bg_white", } secondary_log_colors = {} assert main_log.handlers for handler in main_log.handlers: if isinstance(handler, colorlog.StreamHandler): if isinstance(handler.formatter, colorlog.ColoredFormatter): formatter = handler.formatter assert formatter._fmt == simplelogging.DEFAULT_CONSOLE_FORMAT assert formatter.datefmt is None assert formatter.log_colors == log_colors assert formatter.secondary_log_colors == secondary_log_colors assert isinstance(formatter._style, logging._STYLES["%"][0]) break else: assert False, "No good console formatter found"
def test_default_logger_file(): """Test default logger level file handler presence""" main_log = simplelogging.get_logger("__main__") assert main_log.handlers for handler in main_log.handlers: if isinstance(handler, logging.handlers.RotatingFileHandler): assert False, "A file handler is found, but no file_name provided"
def main(): """Entry point.""" global log parser = argparse.ArgumentParser(description="Markdown to PDF converter") parser.add_argument("-v", "--verbose", action="count", default=0, help="increase verbosity level") parser.add_argument("-t", "--title", help="title of the document", default="") parser.add_argument("-a", "--author", help="author of the document", default="") parser.add_argument( "-s", "--size", help="size of the document", default="A4", choices=["A4", "A5"], ) files = parser.add_mutually_exclusive_group(required=True) files.add_argument( "-i", "--input-path", metavar="PATH", type=str, help="path of the markdown files to convert", default=[], # allow the user to provide no path at all, # this helps writing scripts nargs="+", ) # TODO file on the web, with requests files.add_argument("--version", action="store_true", help="return version") args = parser.parse_args() if args.version: print(pkg_resources.get_distribution("md_to_pdf").version) sys.exit(0) log = simplelogging.get_logger("__main__") if args.verbose < 1: log.reduced_logging() elif args.verbose < 2: log.normal_logging() else: log.full_logging() for md_path in args.input_path: md_to_pdf(md_path, author=args.author, title=args.title, size=args.size)
def test_default_logger_console(): """Test default logger level stream handler presence""" main_log = simplelogging.get_logger("__main__") assert main_log.handlers for handler in main_log.handlers: if isinstance(handler, colorlog.StreamHandler): break else: assert False, "No console handler found"
def test_default_logger_console_level(): """Test default logger level stream formatter level""" main_log = simplelogging.get_logger("__main__") assert main_log.handlers for handler in main_log.handlers: if isinstance(handler, colorlog.StreamHandler): if handler.level == simplelogging.INFO: break else: assert False, "Not good console level"
def test_console_format(): """Test that console is properly configured""" format = "Vincent Poulailleau" main_log = simplelogging.get_logger("__main__", console_format=format) assert main_log.handlers for handler in main_log.handlers: if isinstance(handler, colorlog.StreamHandler): if isinstance(handler.formatter, colorlog.ColoredFormatter): if handler.formatter._fmt == format: break else: assert False, "no handler found with the good format"
import pandas as pd import simplelogging from bokeh.models import ( ColumnDataSource, Whisker, HoverTool, Band, ) from bokeh.plotting import figure, output_file, show from nptdms import TdmsFile from scipy import integrate # logging CONSOLE_FORMAT = " %(log_color)s%(message)s%(reset)s" log = simplelogging.get_logger(console_format=CONSOLE_FORMAT) def create_arg_parser(): """"Creates and returns the ArgumentParser object.""" parser = argparse.ArgumentParser( description="Computation and Graphing of TDMS CO2 Data." ) parser.add_argument("--data", type=str, help="Path to the data directory.") return parser def get_output(path): ABS_PATH = os.path.abspath(path) CSV_PATH = os.path.join(ABS_PATH, "CSV DATA") PLOT_PATH = os.path.join(ABS_PATH, "PLOT DATA")
import simplelogging log = simplelogging.get_logger() def log_some_messages(): log.debug("## some debug ##") log.info("## some info ##") log.warning("## some warning ##") log.error("## some error ##")
import argparse import datetime import hashlib import os import pathlib import re import shutil import subprocess from multiprocessing import Pool import simplelogging logger = simplelogging.get_logger(console_level=simplelogging.INFO) def compute_sha1(filepath): BUF_SIZE = 65536 sha1 = hashlib.sha1() with open(filepath, "rb") as f: while True: data = f.read(BUF_SIZE) if not data: break sha1.update(data) return sha1.hexdigest() def extract_date(filepath): date = None if filepath.lower().endswith((".jpg", ".jpeg")): date = extract_date_image(filepath)
def main(): timeout = 1200 wd = Watchdog(timeout) wd.start() IS_SEDOL = False incidence_file = None output_file = None input_file = None output_dir = None process_type = None process_id = None fields = [] formulas = [] field_types = [] configExcel = None fields = None validation = None files = None error_format_match = False if len(sys.argv) < 5: print( "ERROR : faltan argumentos.. <Tipo Proceso> <ID Proceso> <Fichero> <Directorio Destino>" ) else: start_time = time.time() process_type = sys.argv[1] # Pagina en fichero process_id = sys.argv[2] # ID del tipo input_file = sys.argv[3] # Fichero a procesar output_dir = sys.argv[4] # Direccion donde dejar el fichero config_file = sys.argv[5] # Fichero de equivalencias if not os.path.exists(output_dir): print("ERROR : No se encuentra la carpeta de destino: " + output_dir) return if not os.path.exists(input_file): print("ERROR : No se encuentra el fichero a procesar: " + input_file) return now = datetime.datetime.now() input_file_name = os.path.splitext(ntpath.basename(input_file)) log_name = "log_" + input_file_name[0] + "_" + now.strftime( "%d%m%Y%H%M%S") + ".txt" global log log = simplelogging.get_logger(file_name=output_dir + log_name, console=logger_console) IS_SEDOL = True if not pd.isna(re.search("SEDOL", str(process_id))) else False output_file = "output_" + process_type + ".xlsx" if not IS_SEDOL else "output_SEDOL_" + process_type + ".xlsx" incidence_file = "incidence_" + process_type + ".xlsx" if not IS_SEDOL else "incidence_SEDOL_" + process_type + ".xlsx" log.info("Iniciando Proceso") log.info("PID: " + str(os.getpid())) log.info(process_type + " - ( " + process_id + " )") try: configExcel = pd.read_excel(config_file, process_type) configExcel.columns = configExcel.columns.astype(str) if set(['Campos', process_id]).issubset(configExcel.columns): campos = configExcel["Campos"] tipo = configExcel["Tipo"] fields = campos[1:] fields = fields.dropna(how='all') fields = fields.apply(lambda x: x.strip()) allFormulas = configExcel[process_id] formulas = allFormulas[1:len(fields) + 1] field_types = tipo[1:len(fields) + 1] field_types = field_types.apply(lambda x: x.lower() if not pd.isna(x) else x) filter_func = allFormulas[0] validate_field_types(field_types) validate_formulas(formulas) else: raise Exception( "No se han encontrado las columnas necesarias : (Campos ," + process_id + ")") except OSError as e: log.error("Error: " + str(e)) print("ERROR : Fichero de equivalencias no encontrado") log.error("ERROR : Fichero de equivalencias no encontrado") return except Exception as e: log.error("Error: " + str(e)) print("ERROR : Fichero de equivalencias ERRONEO") log.error("ERROR : Fichero de equivalencias ERRONEO") return log.info("Procesando fichero: " + input_file) file_extension = os.path.splitext(input_file) dataExcel = None try: if (file_extension[1] == ".csv"): dataExcel = pd.read_csv(input_file, header=None, encoding='latin-1', sep=None, engine='python') else: app = xw.App(visible=False) wb = xw.Book(input_file, ignore_read_only_recommended=True) active_sheet_name = wb.sheets.active.name wb.close() app.quit() dataExcel = pd.read_excel(input_file, sheet_name=active_sheet_name, header=None, parse_dates=True, dayfirst=False) dataExcel = dataExcel.replace('\n', '', regex=True).replace('\r', '', regex=True) except Exception as e: print("ERROR : No se pudo abrir el fichero a procesar. " + str(e)) log.error("ERROR : No se pudo abrir el fichero a procesar. " + str(e)) return #print(dataExcel) firstLineExists = gather_data_from_excel(dataExcel, filter_func, fields, formulas, field_types, IS_SEDOL, True) outputData = [] incidenceData = [] if ((len(firstLineExists["outputData"]) > 0 or len(firstLineExists["incidenceData"]) > 0) and firstLineExists["error_format_match"] == False): if (not pd.isna(filter_func)): filter_data_excel(filter_func, dataExcel) log.info("Filtro aplicado.") dataFromExcel = gather_data_from_excel(dataExcel, filter_func, fields, formulas, field_types, IS_SEDOL, False) outputData = dataFromExcel["outputData"] incidenceData = dataFromExcel["incidenceData"] error_format_match = False else: error_format_match = True try: flag_ok = False if (len(outputData) > 0 or len(incidenceData) > 0): resultExcel = pd.DataFrame(outputData, columns=fields) resultExcel.drop_duplicates(keep='first', inplace=True) #print(resultExcel) number_fields = {} for i in range(len(field_types)): if (field_types[i + 1] == 'number'): number_fields[fields[i + 1]] = 2 resultExcel = resultExcel.round(number_fields) ratio_fields = {} for i in range(len(field_types)): if (field_types[i + 1] == 'ratio'): ratio_fields[fields[i + 1]] = 12 resultExcel = resultExcel.round(ratio_fields) #print(resultExcel.dtypes) #print(resultExcel) historyExcel = load_excel(output_dir + output_file, fields) #print(historyExcel.dtypes) #print(historyExcel) finalExcel = None equalRows = pd.DataFrame(columns=['ISIN']) if (not historyExcel.empty and not IS_SEDOL): historyExcel = historyExcel.dropna(how='all') match_dataframes_types(resultExcel, historyExcel) resultExcel = dataframe_difference(resultExcel, historyExcel, "left_only") #print(resultExcel) finalExcel = pd.concat([historyExcel, resultExcel]) #print(finalExcel) else: finalExcel = resultExcel #print(finalExcel) finalExcel.drop_duplicates(keep='first', inplace=True) finalExcel.reset_index(drop=True, inplace=True) #print(finalExcel) columnId = fields[1] if not IS_SEDOL else fields[2] equalRows = finalExcel[finalExcel.duplicated(columnId, keep=False)] equalRows.reset_index(drop=True, inplace=True) #print(equalRows) columnId = fields[1] if not IS_SEDOL else fields[2] finalExcel.drop_duplicates(columnId, inplace=True, keep=False) finalExcel.reset_index(drop=True, inplace=True) #print(finalExcel) #print(resultExcel) newsExcel = dataframe_difference(resultExcel, finalExcel, "both") #print(newsExcel) flag_data = False if (save_excel(output_dir + output_file, finalExcel)): log.info("Guardando fichero: " + output_dir + output_file) log.info("Proceso finalizado...") if (not finalExcel.empty): res_ids = newsExcel.iloc[:, 0] if not IS_SEDOL else newsExcel.iloc[:, 1] if (len(res_ids) > 0): print("OK") log.info("OK") flag_ok = True for ids in res_ids: print(str(ids) + ",OK," + output_file) log.info(str(ids) + ",OK," + output_file) flag_data = True else: print( "ERROR : Fallo en la escritura del fichero de salida.") log.error( "ERROR : Fallo en la escritura del fichero de salida.") return if (len(incidenceData) > 0 or not equalRows.empty): incidenceExcel = pd.DataFrame(incidenceData, columns=fields) #print(equalRows) incidenceExcel = pd.concat([incidenceExcel, equalRows]) #print(incidenceExcel) historyIncidenceExcel = load_excel( output_dir + incidence_file, fields) finalIncidenceExcel = None if (not historyIncidenceExcel.empty and not IS_SEDOL): incidenceExcel = incidenceExcel.where( ~incidenceExcel.notnull(), incidenceExcel.astype('str')) historyIncidenceExcel = historyIncidenceExcel.where( ~historyIncidenceExcel.notnull(), historyIncidenceExcel.astype('str')) #match_dataframes_types(incidenceExcel,historyIncidenceExcel) finalIncidenceExcel = historyIncidenceExcel.append( incidenceExcel) else: finalIncidenceExcel = incidenceExcel finalIncidenceExcel.drop_duplicates( inplace=True) # Los identicos se ignoran finalIncidenceExcel.reset_index(drop=True, inplace=True) #print(finalIncidenceExcel) newsIncidenceExcel = finalIncidenceExcel.append( historyIncidenceExcel) newsIncidenceExcel.drop_duplicates(keep=False, inplace=True) #print(newsIncidenceExcel) if (save_excel(output_dir + incidence_file, finalIncidenceExcel)): log.info("Guardando fichero: " + output_dir + incidence_file) err_isins = newsIncidenceExcel.iloc[:, 0] if (len(err_isins) > 0): if (flag_ok == False): print("OK") log.info("OK") for isin in err_isins: if (not pd.isna(isin)): print(isin + ",ERROR," + incidence_file) log.info(isin + ",ERROR," + incidence_file) flag_data = True else: print( "ERROR : Fallo en la escritura del fichero de incidencias." ) log.error( "ERROR : Fallo en la escritura del fichero de incidencias." ) if (flag_data == False): print("NO DATA: No se encontraron datos para procesar") log.info("NO DATA: No se encontraron datos para procesar") else: if (error_format_match == True): print( "ERROR : El formato para este excel no es correcto: " + process_type + " - " + process_id) log.error( "ERROR : El formato para este excel no es correcto: " + process_type + " - " + process_id) elif (len(outputData) == 0 and len(incidenceData) == 0): print("NO DATA: No se encontraron datos para procesar") log.error("NO DATA: No se encontraron datos para procesar") else: print("ERROR") log.error("ERROR") except (InvalidFormat) as e: print("ERROR : " + str(e)) log.error("ERROR : " + str(e)) except Exception as e: log.error(e) print("ERROR : Ha ocurrido un error inesperado.") log.error("ERROR : Ha ocurrido un error inesperado.") elapsed_time = time.time() - start_time log.info("Tiempo de ejecuciĆ³n: " + str(elapsed_time) + " segundos")
# auto refresing csv of results open and retain past runs copyfile(ffname, "data/pcy_results/pcy_result_0.csv") #Generating items-singletons #def __main__(): if __name__ == '__main__': # global itemCountDict # global hash # cls() # data_lines = open(fileName).readlines() # simplelogger log = simplelogging.get_logger(console_level=-simplelogging.DEBUG, file_name="log/pcy_log.log", console=False) # Testing sets # chunk_percent = [0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1] chunk_percent = [ 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1 ] thresholds = [0.01, 0.05, 0.1] log.debug("START debug session") # basket count data_lines = open(fileName).readlines() basket_count = len(data_lines)
def main(): """Entry point.""" global log parser = argparse.ArgumentParser(description="Linter for *.po files.") parser.add_argument("-v", "--verbose", action="count", default=0) files = parser.add_mutually_exclusive_group(required=True) files.add_argument( "-i", "--input-path", metavar="PATH", type=str, help="path of the file or directory to check", default=[], # allow the user to provide no path at all, # this helps writing scripts nargs="*", ) files.add_argument( "-g", "--github", metavar="python/python-docs-fr/pull/978", type=str, help="path of pull request in GitHub to check", default="", ) files.add_argument( "-p", "--python-docs-fr", metavar="978", type=int, help="ID of pull request in python-docs-fr repository", default=0, ) files.add_argument("--version", action="store_true", help="Return version") parser.add_argument("-c", "--color", action="store_true", help="color output") for checker in checkers: checker.add_arguments(parser) args = parser.parse_args() if args.version: print(pkg_resources.get_distribution("padpo").version) sys.exit(0) if args.color: console_format = ("%(log_color)s[%(levelname)-8s]%(reset)s " "%(green)s%(pofile)s:%(poline)s: " "%(cyan)s[%(checker)s] %(message)s%(reset)s") else: console_format = "%(pofile)s:%(poline)s: %(leveldesc)s: %(message)s" log = simplelogging.get_logger("__main__", console_format=console_format) if args.verbose < 1: log.reduced_logging() elif args.verbose < 2: log.normal_logging() else: log.full_logging() if args.github or args.python_docs_fr: pull_request = "" if args.github: pull_request = args.github if args.python_docs_fr: pull_request = f"python/python-docs-fr/pull/{args.python_docs_fr}" pull_request_info = pull_request_files(pull_request) path = [pull_request_info.download_directory] else: path = args.input_path pull_request_info = None for checker in checkers: checker.configure(args) errors, warnings = check_paths(path, pull_request_info=pull_request_info) if errors: sys.exit(1)
import simplelogging import json import pandas from datetime import datetime from shutil import copy # TODO: to remove the URL column and make API column linkable I'd have to build the indivdiual tables by hand. # TODO: or maybe use soup to edit every place a <tr><td><ADD LINK HERE>name</a> occurs # TODO: or use the formmater in to_html to edit on export # TODO: add group by api # TODO: Integrate `API Value` and `Description` from https://www.dynatrace.com/support/help/shortlink/api-authentication#token-permissions USE_LOCAL_OBJECTS = False log = simplelogging.get_logger( logger_level=simplelogging.DEBUG, console=True, console_level=simplelogging.DEBUG, file_name="log.log") def main(): startTime = datetime.utcnow() # Get the inital page (used for scraping the menu) if USE_LOCAL_OBJECTS: with open('dynatrace_soup.html', 'r') as fp: dynatrace_soup = BeautifulSoup(fp.read(), features="html.parser") log.debug( f"Loaded soup from local storage. Object type: {type(dynatrace_soup)}") else: dynatrace_soup = get_api_start_doc()
def test_default_imported_logger(): """Test default logger of imported module has no handlers""" log = simplelogging.get_logger() assert not log.handlers
revision = '1.1' about = """ Revision: 1.00 License: GPL-3 """ DEBUG = False LOG_CONSOLE_FORMAT = "%(log_color)s%(asctime)s [%(levelname)-8s] %(filename)20s(%(lineno)3s):: %(message)s%(reset)s" LOG_FILE_FORMAT = "%(asctime)s [%(levelname)-8s] %(filename)20s(%(lineno)3s):: %(message)s" if DEBUG: log = simplelogging.get_logger(file_format=LOG_FILE_FORMAT, console_format=LOG_CONSOLE_FORMAT, file_name='log.log') else: log = simplelogging.get_logger(file_format=LOG_FILE_FORMAT, console_format=LOG_CONSOLE_FORMAT) nord = NordVPN() class AppWindow(QMainWindow, Ui_Dialog): def center(self): framegm = self.frameGeometry() centrepoint = QDesktopWidget().availableGeometry().center() framegm.moveCenter(centrepoint) self.move(framegm.topLeft())
def test_disable_console(): """Test that console is disabled if console is False""" main_log = simplelogging.get_logger("__main__", console=False) for handler in main_log.handlers: if isinstance(handler, colorlog.StreamHandler): assert handler._name not in logging._handlers
def test_default_logger_level(): """Test default logger level""" main_log = simplelogging.get_logger("__main__") assert main_log.getEffectiveLevel() == simplelogging.DEBUG
def test_logger_is_from_logging(): """Test logger class""" log = simplelogging.get_logger() assert isinstance(log, logging.Logger)