def setup_logfile(filename, level=DEBUG, maxlevel=None): if filename in logfiles: return logfiles.add(filename) logger = logging.getLogger() handler = FileHandler(filename, encoding='utf-8') add_common_filters(handler) handler.setLevel(level) fmt = LogFormatter( fmt="%(asctime)s [%(name)s] %(levelname)s: %(message)s", output_markers=(START_MARKER, END_MARKER)) handler.setFormatter(fmt) if maxlevel: filt = MaxFilter(maxlevel) handler.addFilter(filt) logger.addHandler(handler) logger.setLevel(min(logger.level, level)) if cache_handler is not None: cache_handler.replay(handler) return handler
def addHandler(handler=None, stream=None, filename=None, filemode='a', format=None, datefmt=None, level=None, max_level=None, filters=(), logger=None): """stream, filename, filemode, format, datefmt: as per logging.basicConfig handler: use a precreated handler instead of creating a new one logging: logging to add the handler to (uses root logging if none specified) filters: an iterable of filters to add to the handler level: only messages of this level and above will be processed max_level: only messages of this level and below will be processed """ # Create the handler if one hasn't been passed in if handler is None: if filename is not None: handler = FileHandler(filename, filemode) else: handler = StreamHandler(stream) # Set up the formatting of the log messages # New API, so it can default to str.format instead of %-formatting formatter = Formatter(format, datefmt) handler.setFormatter(formatter) # Set up filtering of which messages to handle if level is not None: handler.setLevel(level) if max_level is not None: handler.addFilter(LowPassFilter(max_level)) for filter in filters: handler.addFilter(filter) # Add the fully configured handler to the specified logging if logger is None: logger = getLogger() logger.addHandler(handler) return handler
def get_app(static=None, testing=False): """ factory for app """ if static: #static var return static app = Flask(__name__) init_app(app) static = app #debug mode static file serving if app.config['DEBUG']: from werkzeug.wsgi import SharedDataMiddleware import os app.wsgi_app = SharedDataMiddleware(app.wsgi_app, { '/': os.path.join(os.path.dirname(__file__), 'static') }) #email logger if not app.debug: import logging from logging import FileHandler file_handler = FileHandler(LOG_FILE) file_handler.setLevel(logging.INFO) app.logger.addHandler(file_handler) if not testing: @app.teardown_request def shutdown_session(exception=None): from db import db db.session.remove() return app
def create_app(config_name): application = Flask(__name__) cfg = config[config_name] application.config.from_object(cfg) handler = FileHandler('errors.log') handler.setLevel(logging.DEBUG) handler.setFormatter(logging.Formatter( '%(asctime)s %(levelname)s: %(message)s ' '[in %(pathname)s:%(lineno)d]' )) application.logger.addHandler(handler) application.logger.info('logging set up') initialize_blueprints(application) application.logger.info('blueprints initialized') initialize_api_resources(application) application.logger.info('api resources initialized') db.init_app(application) application.logger.info('db initialized') admin.init_app(application) admin.name = 'GreenDeploy' admin.template_mode = 'bootstrap3' admin.add_view(ModelView(Project, db.session)) admin.add_view(ModelView(DriveDestination, db.session)) admin.add_view(ModelView(BuildHistory, db.session)) application.logger.info('admin initialized') return application
def register_local_log(self, path, level=None, purge_buffer=True): """The shinken logging wrapper can write to a local file if needed and return the file descriptor so we can avoid to close it. Add logging to a local log-file. The file will be rotated once a day """ self.log_set = True # Todo : Create a config var for backup count if os.path.exists(path) and not stat.S_ISREG(os.stat(path).st_mode): # We don't have a regular file here. Rotate may fail # It can be one of the stat.S_IS* (FIFO? CHR?) handler = FileHandler(path) else: handler = TimedRotatingFileHandler(path, 'midnight', backupCount=5) if level is not None: handler.setLevel(level) if self.name is not None: handler.setFormatter(defaultFormatter_named) else: handler.setFormatter(defaultFormatter) self.addHandler(handler) # Ok now unstack all previous logs if purge_buffer: self._destack() # Todo : Do we need this now we use logging? return handler.stream.fileno()
def __setup_logging(app): log_file_path = "" log_dir_path = "" log_level = app.config.get('LOG_LEVEL', logging.INFO) if os.path.isabs(app.config['LOG_DIR']): log_dir_path = app.config['LOG_DIR'] log_file_path = log_dir_path + app.config['LOG_FILE'] else: here = os.path.dirname(os.path.abspath(__file__)) log_dir_path = os.path.join( os.path.dirname(here), app.config['LOG_DIR']) log_file_path = log_dir_path + app.config['LOG_FILE'] if not os.path.isdir(log_dir_path): os.makedirs(log_dir_path, mode=app.config['LOG_FILE_MODE']) if not os.path.isfile(log_file_path): open(log_file_path, 'a').close() log_file_handler = FileHandler(filename=log_file_path, encoding='utf-8') log_file_handler.setLevel(log_level) log_file_handler.setFormatter(Formatter( '[%(asctime)s] [%(levelname)s] %(message)s %(module)s:%(funcName)s:%(lineno)d' )) app.logger.addHandler(log_file_handler) app.logger.setLevel(log_level)
def add_disk_handler(prefix, level=logging.NOTSET): """ Enable typical logging to disk. """ # generate an unused log file path from os.path import lexists from itertools import count for i in count(): path = "%s.%i" % (prefix, i) if not lexists(path): break # build a handler from cargo.temporal import utc_now handler = FileHandler(path, encoding="utf-8") handler.setFormatter(VerboseFileFormatter()) handler.setLevel(level) # add it logging.root.addHandler(handler) log.debug("added log handler for file %s at %s", path, utc_now()) return handler
class CaptureLog: """Context to capture log from a specific logger and write it to a file Parameters ---------- filename : str Where to write the log file. mode : str Mode for opening the log file (default 'w'). name : str Name of the logger from which to capture (default 'mne'). """ def __init__(self, filename, mode='w', logger='mne', level='debug'): self.logger = logger self.level = log_level(level) self.handler = FileHandler(filename, mode) self.handler.setLevel(self.level) self.handler.setFormatter(Formatter("%(levelname)-8s :%(message)s")) self._old_level = None def __enter__(self): logger = getLogger(self.logger) logger.addHandler(self.handler) if logger.level == 0 or logger.level > self.level: self._old_level = logger.level logger.setLevel(self.level) return logger def __exit__(self, exc_type, exc_val, exc_tb): self.handler.close() logger = getLogger(self.logger) logger.removeHandler(self.handler) if self._old_level is not None: logger.setLevel(self._old_level)
def setup_logger(app_name): """ Instantiate a logger object Usage: logger = setup_logger('foo') # saved as foo.log logger.info("Some info message") logger.warn("Some warning message") logger.error("Some error message") ... [for more options see: http://docs.python.org/2/library/logging.html] """ logger = getLogger(app_name) logger.setLevel(DEBUG) # create file handler which logs even debug messages fh = FileHandler(app_name + '.log') fh.setLevel(DEBUG) # create console handler with a higher log level ch = StreamHandler() ch.setLevel(ERROR) # create formatter and add it to the handlers formatter = Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) return logger
def create_app(config_name): global app app.config.from_object(config[config_name]) # Setup database # Currently inits mongoDB init_db(app) # Todo make intializing blueprints consistent app.register_blueprint(bp_index) app.register_blueprint(bp_auth) app.register_blueprint(bp_timer) app.register_blueprint(v1_api) init_api(app) file_handler = FileHandler("flask.log") file_handler.setLevel(logging.INFO) app.logger.addHandler(file_handler) config[config_name].init_app(app) # init_flask_security(app) mail = Mail(app) return app
def start(self): from logging import getLogger, FileHandler, Formatter, DEBUG self.log = getLogger(self.name) self.log.setLevel(DEBUG) formatter = Formatter( '%(asctime)s [%(process)d:%(threadName)s] %(levelname)-8s %(name)s: %(message)s') # Unique log handler (single file) handler = FileHandler(self.uniquefile, "w") handler.setLevel(DEBUG) handler.setFormatter(formatter) self.log.addHandler(handler) # If you suspect that the diff stuff isn't working, un comment the next # line. You should see this show up once per-process. # self.log.info("Here is a line that should only be in the first output.") # Setup output used for testing handler = self.getLogHandler(self.sharedfile) handler.setLevel(DEBUG) handler.setFormatter(formatter) self.log.addHandler(handler) # If this ever becomes a real "Thread", then remove this line: self.run()
def create_app(self): self.twill = Twill(app) file_handler = FileHandler('loggingFile') file_handler.setLevel(logging.DEBUG) app.logger.addHandler(file_handler) app.logger.debug('WTFCAKES') return app
class Logging(logging.FileHandler): @classmethod def __init__(self,user_connect_=None): self.user_connect = user_connect_ self.application = app\ @classmethod def create_login_info(self,user,time,url): self.logging = FileHandler('icollect_info.log') self.logging.setLevel(logging.DEBUG) self.logging.setFormatter(Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) self.application.logger.addHandler(self.logging) create_dict_to_loggin_info = dict({'user_connect':user,'time':time,'url':url}) self.application.logger.info('Info LogIn' + ":" + str(create_dict_to_loggin_info)) @classmethod def create_logout_info(self,user,time): self.logging = FileHandler('icollect_info.log') self.logging.setLevel(logging.DEBUG) self.logging.setFormatter(Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) self.application.logger.addHandler(self.logging) create_dict_to_loggin_info = dict({'user_connect':user,'time':time}) self.application.logger.info('Info Logout' + ":" + str(create_dict_to_loggin_info))
def start_logging(): global logfocus, file_handler from lmfdb.utils.config import Configuration config = Configuration() logging_options = config.get_logging() file_handler = FileHandler(logging_options['logfile']) file_handler.setLevel(WARNING) if 'logfocus' in logging_options: logfocus = logging_options['logfocus'] getLogger(logfocus).setLevel(DEBUG) root_logger = getLogger() root_logger.setLevel(INFO) root_logger.name = "LMFDB" formatter = Formatter(LmfdbFormatter.fmtString.split(r'[')[0]) ch = StreamHandler() ch.setFormatter(formatter) root_logger.addHandler(ch) cfg = config.get_all() if "postgresql_options" and "password" in cfg["postgresql_options"]: cfg["postgresql_options"]["password"] = "******" info("Configuration = {}".format(cfg) ) check_sage_version()
def root_doc(args, l, rc): from ambry.ui import app, configure_application, setup_logging import ambry.ui.views as views import os import logging from logging import FileHandler import webbrowser port = args.port if args.port else 8085 cache_dir = l._doc_cache.path('', missing_ok=True) config = configure_application(dict(port=port)) file_handler = FileHandler(os.path.join(cache_dir, "web.log")) file_handler.setLevel(logging.WARNING) app.logger.addHandler(file_handler) print 'Serving documentation for cache: ', cache_dir if not args.debug: # Don't open the browser on debugging, or it will re-open on every # application reload webbrowser.open("http://localhost:{}/".format(port)) app.run(host=config['host'], port=int(port), debug=args.debug)
def create_app(config_name): app.config.from_object(config[config_name]) db.init_app(app) login_manager.init_app(app) login_manager.session_protection = 'strong' login_manager.login_view = 'admin.login' if not app.debug: import logging from logging import FileHandler, Formatter file_handler = FileHandler(Constant.LOG_DIR, encoding='utf8') file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(Formatter( '[%(asctime)s] %(levelname)s: %(message)s ' '[in %(pathname)s:%(lineno)d]')) app.logger.addHandler(file_handler) from main import main as main_blueprint app.register_blueprint(main_blueprint) from admin import admin as admin_blueprint app.register_blueprint(admin_blueprint, url_prefix='/admin') patch_request_class(app, size=16*1024*1024) # 16MB configure_uploads(app, resource_uploader) return app
def init_logging(): # Disable default stderr handler root = getLogger().addHandler(logging.NullHandler()) # Get the loggers used in pytelemetry.telemetry.telemetry file rx = getLogger("telemetry.rx") tx = getLogger("telemetry.tx") rx.setLevel(logging.DEBUG) tx.setLevel(logging.DEBUG) # Format how data will be .. formatted formatter = logging.Formatter('%(asctime)s | %(levelname)s | %(message)s') # Create a handler to save logging output to a file dateTag = datetime.datetime.now().strftime("%Y-%b-%d_%H-%M-%S") in_handler = FileHandler('in-%s.log' % dateTag) in_handler.setLevel(logging.DEBUG) # Also pass all messages in_handler.setFormatter(formatter) out_handler = FileHandler('out-%s.log' % dateTag) out_handler.setLevel(logging.DEBUG) # Also pass all messages out_handler.setFormatter(formatter) # Attach the logger to the handler rx.addHandler(in_handler) tx.addHandler(out_handler)
def init_app_logger(app): file_handler = FileHandler('flask.log') file_handler.setFormatter(Formatter( '%(asctime)s|%(levelname)s|%(pathname)s:%(lineno)d|%(message)s' )) file_handler.setLevel(logging.INFO) app.logger.addHandler(file_handler)
def configure_logging(app, filename): """Configure app to log to that file.""" handler = FileHandler(filename, encoding='utf-8') handler.setFormatter(Formatter('%(asctime)s %(funcName)s %(message)s')) handler.setLevel(logging.INFO) app.logger.addHandler(handler)
def actionWork(self, *args, **kwargs): """Performing the set of actions""" nextinput = args #set the logger to save the tasklog formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s:%(message)s") taskdirname = "logs/tasks/%s/" % self._task['tm_username'] if not os.path.isdir(taskdirname): os.mkdir(taskdirname) taskhandler = FileHandler(taskdirname + self._task['tm_taskname'] + '.log') taskhandler.setLevel(logging.DEBUG) self.logger.addHandler(taskhandler) for work in self.getWorks(): self.logger.debug("Starting %s on %s" % (str(work), self._task['tm_taskname'])) t0 = time.time() try: output = work.execute(nextinput, task=self._task) except StopHandler, sh: msg = "Controlled stop of handler for %s on %s " % (self._task, str(sh)) self.logger.error(msg) nextinput = Result(task=self._task, result='StopHandler exception received, controlled stop') break #exit normally. Worker will not notice there was an error except TaskWorkerException, twe: self.logger.debug(str(traceback.format_exc())) #print the stacktrace only in debug mode raise WorkerHandlerException(str(twe)) #TaskWorker error, do not add traceback to the error propagated to the REST
def configure_loggers(log, verbosity, log_file, log_verbosity): LOGFMT_CONSOLE = ( "[%(asctime)s] %(name)-10s %(levelname)-7s in %(module)s.%(funcName)s()," " line %(lineno)d\n\t%(message)s" ) LOGFMT_FILE = ( "[%(asctime)s] [%(process)d]%(name)-10s %(levelname)-7s in %(module)s.%(funcName)s()," " line %(lineno)d\n\t%(message)s" ) # Configure root logger to log to stdout logging.basicConfig(level=verbosity, datefmt="%H:%M:%S", format=LOGFMT_CONSOLE) # Configure main logger to rotate log files rh = RotatingFileHandler(log_file, maxBytes=100000, backupCount=25) log.addHandler(rh) # Configure main logger to log to a file if log_file: fh = FileHandler(log_file, "w") fh.setFormatter(Formatter(LOGFMT_FILE, "%Y-%m-%d %H:%M:%S")) fh.setLevel(log_verbosity) log.addHandler(fh) return log
def setup_logging(): log_formatter = Formatter( '''{"message_type":"%(levelname)s","location":"%(pathname)s","line_number":%(lineno)d,"module":"%(module)s","function":"%(funcName)s","time":"%(asctime)s","message":"%(message)s"}''') # pylint: disable=C0301 fh = FileHandler('flask_logs.log') fh.setLevel(INFO) fh.setFormatter(log_formatter) application.logger.addHandler(fh) application.logger.setLevel(INFO) if not application.debug: from application.models import Administrator dbadmins = Administrator.query.all() if dbadmins is not None: emails = [dbadmin.email for dbadmin in dbadmins] emailErrorHandler = TlsSMTPHandler( ('smtp.gmail.com', 587), '*****@*****.**', emails, 'Server Error', ('*****@*****.**', 'ANRISNTPTV') ) emailErrorHandler.setFormatter(Formatter( ''' Message type: %(levelname)s Location: %(pathname)s:%(lineno)d Module: %(module)s Function: %(funcName)s Time: %(asctime)s Message: %(message)s ''')) application.logger.addHandler(emailErrorHandler)
def init_app(app, name=''): """ Configures the provided app's logger. :param app: the application object to configure the logger :param name: the name of the logger to create and configure """ # flask app object automatically registers its own debug logger if # app.debug is True. Remove it becuase debug logging is handled here # instead. del app.logger.handlers[:] log_path = app.config['LOG_PATH'] log_level = app.config['LOG_LEVEL'] or '' log_filter = app.config['LOG_FILTER'] log_ignore = app.config['LOG_IGNORE'] handler = FileHandler(log_path) if log_path else StreamHandler() handler.setLevel(log_level.upper() or ('DEBUG' if app.debug else 'WARNING')) # noqa handler.addFilter(MultiNameFilter(log_filter, log_ignore)) handler.setFormatter(Formatter( '%(asctime)s %(process)s %(thread)-15s %(name)-10s %(levelname)-8s %(message)s', # noqa '%H:%M:%S' if app.debug else '%Y-%m-%d %H:%M:%S%z')) logger = getLogger(name) logger.setLevel(handler.level) logger.addHandler(handler)
def log_server(level, queue, filename, mode='w'): """Run the logging server. This listens to the queue of log messages, and handles them using Python's logging handlers. It prints to stderr, as well as to a specified file, if it is given. """ formatter = _get_formatter() handlers = [] sh = StreamHandler() sh.setFormatter(formatter) sh.setLevel(level) handlers.append(sh) if filename: fh = FileHandler(filename, mode) fh.setFormatter(formatter) fh.setLevel(level) handlers.append(fh) listener = QueueListener(queue, *handlers) listener.start() # For some reason, queuelisteners run on a separate thread, so now we just # "busy wait" until terminated. try: while True: time.sleep(1) except KeyboardInterrupt: pass finally: listener.stop()
def get_logger(log_name = 'default.log', log_level = DEBUG): formatter = Formatter('[%(asctime)s][%(process)d][%(filename)s:%(lineno)s][%(levelname)s]: %(message)s') logger = getLogger('%s.%s' % (log_name.replace('/', '.'), time.time())) logger.handlers = [] logger.setLevel(DEBUG) logger.propagate = False console = StreamHandler() console.setFormatter(formatter) console.setLevel(WARN) logger.addHandler(console) logfile = os.path.dirname(os.path.abspath(sys.argv[0])) logfile = os.path.join(logfile, log_name) logfiledebug = FileHandler(filename = logfile, mode='a') logfiledebug.setFormatter(formatter) logfiledebug.setLevel(log_level) logger.addHandler(logfiledebug) def _logger_die(logger, msg): logger.error(msg) raise AssertionError(msg) logger.die = lambda msg: _logger_die(logger, msg) return logger
def setHandler(logger, lvl, path, _format): """ Set right handler related to input lvl, path and format. :param Logger logger: logger on which add an handler. :param str lvl: logging level. :param str path: file path. :param str _format: logging message format. """ class _Filter(Filter): """Ensure message will be given for specific lvl""" def filter(self, record): return record.levelname == lvl # get the rights formatter and filter to set on a file handler handler = FileHandler(path) handler.addFilter(_Filter()) handler.setLevel(lvl) formatter = Formatter(_format) handler.setFormatter(formatter) # if an old handler exist, remove it from logger if hasattr(logger, lvl): old_handler = getattr(logger, lvl) logger.removeHandler(old_handler) logger.addHandler(handler) setattr(logger, lvl, handler)
def configureOutput(logger_p, path='./Log.txt'): """ configures the logging """ # if no handlers have been configured... if 0 == len(logger_p.handlers): # let everything pass by default logger_p.setLevel(logging.DEBUG) # this will make the project logger write all messages in a file file_handler = FileHandler(path, mode = 'w') file_handler.setFormatter(Formatter('%(levelname)-8s : %(message)s')) # accept even the most trivial shit file_handler.setLevel(logging.DEBUG) logger_p.addHandler(file_handler) # this will make the logger write INFO messages to the console console_handler = StreamHandler() console_handler.setFormatter(Formatter('%(message)s')) #ignore debug messages, only Info, warnings, errors and criticals console_handler.setLevel(logging.INFO) logger_p.addHandler(console_handler) else: pass
def _setup_task_logger(logger): """Configure a task logger to generate site- and task-specific logs.""" if logger.handlers: # Already processed return parts = logger.name.split(".") if len(parts) < 4: # Malformed return site = parts[2] task = parts[3] _ensure_dirs(os.path.join(_log_dir, site)) formatter = Formatter( fmt="[%(asctime)s %(levelname)-7s] %(message)s", datefmt=_DATE_FORMAT) infohandler = TimedRotatingFileHandler( os.path.join(_log_dir, site, task + ".log"), "midnight", 1, 30) infohandler.setLevel("INFO") debughandler = FileHandler( os.path.join(_log_dir, site, task + ".log.verbose"), "w") debughandler.setLevel("DEBUG") errorhandler = RotatingFileHandler( os.path.join(_log_dir, site, task + ".err"), maxBytes=1024**2, backupCount=4) errorhandler.setLevel("WARNING") for handler in [infohandler, debughandler, errorhandler]: handler.setFormatter(formatter) logger.addHandler(handler)
def actionWork(self, *args, **kwargs): """Performing the set of actions""" nextinput = args #set the logger to save the tasklog formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s:%(message)s") taskdirname = "logs/tasks/%s/" % self._task['tm_username'] if not os.path.isdir(taskdirname): os.mkdir(taskdirname) taskhandler = FileHandler(taskdirname + self._task['tm_taskname'] + '.log') taskhandler.setLevel(logging.DEBUG) self.logger.addHandler(taskhandler) for work in self.getWorks(): self.logger.debug("Starting %s on %s" % (str(work), self._task['tm_taskname'])) t0 = time.time() try: output = work.execute(nextinput, task=self._task) except StopHandler as sh: msg = "Controlled stop of handler for %s on %s " % (self._task, str(sh)) self.logger.error(msg) nextinput = Result(task=self._task, result='StopHandler exception received, controlled stop') break #exit normally. Worker will not notice there was an error except TaskWorkerException as twe: self.logger.debug(str(traceback.format_exc())) #print the stacktrace only in debug mode self.removeTaskLogHandler(taskhandler) raise WorkerHandlerException(str(twe)) #TaskWorker error, do not add traceback to the error propagated to the REST except Exception as exc: msg = "Problem handling %s because of %s failure, traceback follows\n" % (self._task['tm_taskname'], str(exc)) msg += str(traceback.format_exc()) self.logger.error(msg) self.removeTaskLogHandler(taskhandler) raise WorkerHandlerException(msg) #Errors not foreseen. Print everything! finally: #upload logfile of the task to the crabcache logpath = 'logs/tasks/%s/%s.log' % (self._task['tm_username'], self._task['tm_taskname']) if os.path.isfile(logpath) and 'user_proxy' in self._task: #the user proxy might not be there if myproxy retrieval failed cacheurldict = {'endpoint': self._task['tm_cache_url'], 'cert' : self._task['user_proxy'], 'key' : self._task['user_proxy']} try: ufc = UserFileCache(cacheurldict) logfilename = self._task['tm_taskname'] + '_TaskWorker.log' ufc.uploadLog(logpath, logfilename) except HTTPException as hte: msg = ("Failed to upload the logfile to %s for task %s. More details in the http headers and body:\n%s\n%s" % (self._task['tm_cache_url'], self._task['tm_taskname'], hte.headers, hte.result)) self.logger.error(msg) except Exception as e: msg = "Unknown error while uploading the logfile for task %s" % self._task['tm_taskname'] self.logger.exception(msg) t1 = time.time() self.logger.info("Finished %s on %s in %d seconds" % (str(work), self._task['tm_taskname'], t1-t0)) try: nextinput = output.result except AttributeError: nextinput = output self.removeTaskLogHandler(taskhandler) return nextinput
endTime = unix_time_millis(datetime.datetime.now()) totalTime = endTime - startTime response_data['vthResponse']['testDuration'] = totalTime response_data['vthResponse']['abstractMessage'] = 'error: ' + str(ex) app.logger.error('ERROR:{}'.format(str(ex))) return jsonify(response_data) #finish up building response endTime = unix_time_millis(datetime.datetime.now()) totalTime = endTime - startTime response_data['vthResponse']['testDuration'] = totalTime if ret_url is not None: sendCallback(ret_url, response_data) return '', 200 return jsonify(response_data), 200 @app.route("/otf/vth/oran/smo/v1/health", methods=['GET']) def getHealth(): return 'UP' if __name__ == '__main__': logHandler = FileHandler('smo-o1-vth.log', mode='a') logHandler.setLevel(logging.INFO) app.logger.setLevel(logging.INFO) app.logger.addHandler(logHandler) # context = ('opt/cert/otf.pem', 'opt/cert/privateKey.pem') # app.run(debug = False, host = '0.0.0.0', port = 5000, ssl_context = context) app.run(debug=False, host='0.0.0.0', port=5000)
def note_and_log(cls): """ This will be used as a decorator on class to activate logging and store messages in the variable cls._notes This will allow quick access to events in the web app. A note can be added to cls._notes without logging if passing the argument log=false to function note() Something can be logged without addind a note using function log() """ if hasattr(cls, "DEBUG_LEVEL"): if cls.DEBUG_LEVEL == "debug": file_level = logging.DEBUG console_level = logging.DEBUG elif cls.DEBUG_LEVEL == "info": file_level = logging.INFO console_level = logging.INFO else: file_level = logging.WARNING console_level = logging.INFO # Notes object cls._notes = namedtuple("_notes", ["timestamp", "notes"]) cls._notes.timestamp = [] cls._notes.notes = [] # Defining log object cls.logname = "{} | {}".format(cls.__module__, cls.__name__) cls._log = logging.getLogger("BAC0_Root.{}.{}".format( cls.__module__, cls.__name__)) # Set level to debug so filter is done by handler cls._log.setLevel(logging.DEBUG) # Console Handler ch = logging.StreamHandler(sys.stderr) ch.set_name("stderr") ch.setLevel(logging.CRITICAL) ch2 = logging.StreamHandler(sys.stdout) ch2.set_name("stdout") ch2.setLevel(console_level) formatter = logging.Formatter("{asctime} - {levelname:<8}| {message}", style="{") # File Handler _PERMISSION_TO_WRITE = True logUserPath = expanduser("~") logSaveFilePath = join(logUserPath, ".BAC0") logFile = join(logSaveFilePath, "BAC0.log") if not os.path.exists(logSaveFilePath): try: os.makedirs(logSaveFilePath) except: _PERMISSION_TO_WRITE = False if _PERMISSION_TO_WRITE: fh = FileHandler(logFile) fh.set_name("file_handler") fh.setLevel(file_level) fh.setFormatter(formatter) ch.setFormatter(formatter) ch2.setFormatter(formatter) # Add handlers the first time only... if not len(cls._log.handlers): if _PERMISSION_TO_WRITE: cls._log.addHandler(fh) cls._log.addHandler(ch) cls._log.addHandler(ch2) LogList.LOGGERS.append(cls._log) def log_title(self, title, args=None, width=35): cls._log.debug("") cls._log.debug("#" * width) cls._log.debug("# {}".format(title)) cls._log.debug("#" * width) if args: cls._log.debug("{!r}".format(args)) cls._log.debug("#" * 35) def log_subtitle(self, subtitle, args=None, width=35): cls._log.debug("") cls._log.debug("=" * width) cls._log.debug("{}".format(subtitle)) cls._log.debug("=" * width) if args: cls._log.debug("{!r}".format(args)) cls._log.debug("=" * width) def log(self, note, *, level=logging.DEBUG): """ Add a log entry...no note """ if not note: raise ValueError("Provide something to log") note = "{} | {}".format(cls.logname, note) cls._log.log(level, note) def note(self, note, *, level=logging.INFO, log=True): """ Add note to the object. By default, the note will also be logged :param note: (str) The note itself :param level: (logging.level) :param log: (boolean) Enable or disable logging of note """ if not note: raise ValueError("Provide something to log") note = "{} | {}".format(cls.logname, note) cls._notes.timestamp.append(datetime.now()) cls._notes.notes.append(note) if log: cls.log(level, note) @property def notes(self): """ Retrieve notes list as a Pandas Series """ if not _PANDAS: return dict(zip(self._notes.timestamp, self._notes.notes)) return pd.Series(self._notes.notes, index=self._notes.timestamp) def clear_notes(self): """ Clear notes object """ cls._notes.timestamp = [] cls._notes.notes = [] # Add the functions to the decorated class cls.clear_notes = clear_notes cls.note = note cls.notes = notes cls.log = log cls.log_title = log_title cls.log_subtitle = log_subtitle return cls
import logging import sys from logging import FileHandler from logging import Formatter LOG_FORMAT = ("%(message)s") LOG_LEVEL = logging.INFO # messaging logger RESULTADOS_LOG_FILE = str(sys.argv[1]) + ".log" resultados_logger = logging.getLogger(str(sys.argv[1])) resultados_logger.setLevel(LOG_LEVEL) resultados_logger_file_handler = FileHandler(RESULTADOS_LOG_FILE) resultados_logger_file_handler.setLevel(LOG_LEVEL) resultados_logger_file_handler.setFormatter(Formatter(LOG_FORMAT)) resultados_logger.addHandler(resultados_logger_file_handler) # payments logger PROGRESSO_LOG_FILE = str(sys.argv[1]) + "_progresso_rodadas.log" progresso_logger = logging.getLogger(str(sys.argv[1]) + "_progresso_rodadas") progresso_logger.setLevel(LOG_LEVEL) progresso_file_handler = FileHandler(PROGRESSO_LOG_FILE) progresso_file_handler.setLevel(LOG_LEVEL) progresso_file_handler.setFormatter(Formatter(LOG_FORMAT)) progresso_logger.addHandler(progresso_file_handler) #1 FASE RESULTADOS_1FASE_LOG_FILE = str(sys.argv[1]) + "_1fase.log"
import time import sqlite3 from tqdm import tqdm # 상수 종목별매수상한 = 1000000 # 종목별매수상한 백만원 매수수수료비율 = 0.00015 # 매도시 평단가에 곱해서 사용 매도수수료비율 = 0.00015 + 0.003 # 매도시 현재가에 곱해서 사용 화면번호 = "1234" # 로그 파일 핸들러 BASE_DIR = os.path.dirname(os.path.abspath(__file__)) fh_log = FileHandler(os.path.join(BASE_DIR, 'logs/debug.log'), encoding='utf-8') fh_log.setLevel(logging.DEBUG) # 로거 생성 및 핸들러 등록 logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) logger.addHandler(fh_log) class SyncRequestDecorator: """키움 API 비동기 함수 데코레이터 """ @staticmethod def kiwoom_sync_request(func): def func_wrapper(self, *args, **kwargs): if kwargs.get('nPrevNext', 0) == 0:
import pandas as pd from urllib.error import HTTPError from urllib.parse import quote from urllib.parse import urlencode logger = getLogger('dev-debug') formatter = Formatter\ ('%(asctime)s [%(levelname)s] [%(filename)s: \ %(funcName)s: %(lineno)d] %(message)s' ) handlerSh = StreamHandler() handlerFile = FileHandler('error.log') handlerSh.setFormatter(formatter) handlerSh.setLevel(DEBUG) handlerFile.setFormatter(formatter) handlerFile.setLevel(DEBUG) logger.setLevel(DEBUG) logger.addHandler(handlerSh) logger.addHandler(handlerFile) logger.debug('log start') # Yelp Fusion no longer uses OAuth as of December 7, 2017. # You no longer need to provide Client ID to fetch Data # It now uses private keys to authenticate requests (API Key) # You can find it on # https://www.yelp.com/developers/v3/manage_app API_KEY = os.environ.get("YELP_API_KEY") # API constants, you shouldn't have to change these. API_HOST = 'https://api.yelp.com' SEARCH_PATH = '/v3/businesses/search'
from spider.CFSpider import CFSpider from spider.HDUSpider import HDUSpider from spider.POJSpider import POJSpider from spider.ZOJSpider import ZOJSpider from spider.UVASpider import UVASpider from spider.BNUSpider import BNUSpider from spider.BCSpider import BCSpider from spider.VJSpider import VJSpider from flask.ext.sqlalchemy import SQLAlchemy from sqlalchemy import or_ app = Flask(__name__) app.config.from_pyfile('config.py') db.init_app(app) file_handler = FileHandler(app.root_path + "/log/spider_errors.log") file_handler.setLevel(logging.ERROR) file_handler.setFormatter( Formatter('%(asctime)s %(levelname)s: %(message)s ' '[in %(pathname)s:%(lineno)d]')) app.logger.addHandler(file_handler) spider_lock = RLock() class AccountUpdateServer(Thread): def __init__(self, oj_name): Thread.__init__(self) self.oj_name = oj_name self.spider_module_name = sys.modules['spider.' + self.oj_name.upper() + 'Spider'] self.spider_class_name = oj_name.upper() + 'Spider' self.spider = getattr(self.spider_module_name,
from flask import Flask, session, redirect, url_for, flash from logging import FileHandler, WARNING from functools import wraps from isbnlib import is_isbn13, to_isbn10, is_isbn10 from sqlalchemy import create_engine from sqlalchemy.orm import scoped_session, sessionmaker # Set up app app = Flask(__name__) # Error Log: if not app.debug: f_handler = FileHandler('error_log.txt') f_handler.setLevel(WARNING) app.logger.addHandler(f_handler) # Change in production app.secret_key = 'secret' # Check for environment variable if not os.getenv("DATABASE_URL"): raise RuntimeError("DATABASE_URL is not set") # Configure session to use filesystem app.config["SESSION_PERMANENT"] = False app.config["SESSION_TYPE"] = "filesystem" # Set up database engine = create_engine(os.getenv("DATABASE_URL"))
best_aa = pd.DataFrame(np.zeros(shape=(20, LEN))) * 1.0 best_nt = pd.DataFrame(np.zeros(shape=(12, LEN))) * 1.0 # ----- initialize best results ------------ # ----- initialize logger ------------ logger = getLogger("IUPAC") logger.setLevel(logging.DEBUG) handler_format = Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') stream_handler = StreamHandler() stream_handler.setLevel(logging.DEBUG) stream_handler.setFormatter(handler_format) file_handler = FileHandler(OUTDIR + 'iupac.log', 'a') file_handler.setLevel(logging.DEBUG) file_handler.setFormatter(handler_format) logger.addHandler(stream_handler) logger.addHandler(file_handler) # ----- initialize logger ------------ #read csv def readcsv(infile): df = pd.read_csv(infile, index_col=0, header=None) return df def find_best_matches():
from logging import Logger, FileHandler, Formatter, WARN, INFO, DEBUG DEBUG = False ### logger config logger = Logger('cls') WARN_LOGGER = "/tmp/cls_warn.log" INFO_LOGGER = "/tmp/cls_info.log" DEBUG_LOGGER = "/tmp/cls_debug.log" formatter = Formatter("%(asctime)s - %(levelname)s - %(message)s") filewarnhandler = FileHandler(WARN_LOGGER, 'a') filewarnhandler.setLevel(WARN) filewarnhandler.setFormatter(formatter) fileinfohandler = FileHandler(INFO_LOGGER, 'a') fileinfohandler.setLevel(INFO) fileinfohandler.setFormatter(formatter) filedebughandler = FileHandler(DEBUG_LOGGER, 'a') filedebughandler.setLevel(DEBUG) filedebughandler.setFormatter(formatter) logger.addHandler(filewarnhandler) logger.addHandler(fileinfohandler) logger.addHandler(filedebughandler) # origin_warning = logger.warning # # # def my_warning(*args, **kwargs):
@app.errorhandler(404) def not_found_error(error): return render_template('errors/404.html'), 404 @app.errorhandler(500) def server_error(error): return render_template('errors/500.html'), 500 if not app.debug: file_handler = FileHandler('error.log') file_handler.setFormatter( Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]') ) app.logger.setLevel(logging.INFO) file_handler.setLevel(logging.INFO) app.logger.addHandler(file_handler) app.logger.info('errors') #----------------------------------------------------------------------------# # Launch. #----------------------------------------------------------------------------# # Default port: if __name__ == '__main__': app.run() # Or specify port manually: ''' if __name__ == '__main__': port = int(os.environ.get('PORT', 5000))
# Test text extraction test_extract_text_from_news(la_republica) # Test filtering test_filter_news_by_keywords(la_republica) del la_republica if __name__ == "__main__": # Configure logger: oddcrawler needsd to be the top logger logger = getLogger('oddcrawler') logger.setLevel(DEBUG) # create file file handler fh = FileHandler('extractor_test.log') fh.setLevel(DEBUG) # create console handler ch = StreamHandler() ch.setLevel(ERROR) # create formatter and add it to handlers formatter = Formatter('%(levelname)s %(asctime)-15s %(message)s') fh.setFormatter(formatter) ch.setFormatter(formatter) logger.addHandler(fh) logger.addHandler(ch) complete_test_for_la_republica() complete_test_for_monumental() complete_test_for_cr_hoy() complete_test_for_la_prensa_libre()
rate_s, rate_v, rate_o ] return list_ret if __name__ == '__main__': from logging import StreamHandler, DEBUG, Formatter, FileHandler log_fmt = Formatter( '%(asctime)s %(name)s %(lineno)d [%(levelname)s][%(funcName)s] %(message)s ' ) handler = FileHandler('svo.py.log', 'w') handler.setLevel(DEBUG) handler.setFormatter(log_fmt) logger.setLevel(DEBUG) logger.addHandler(handler) handler = StreamHandler() handler.setLevel('INFO') handler.setFormatter(log_fmt) logger.setLevel('INFO') logger.addHandler(handler) p = Pool() df = pandas.read_csv('../data/train_clean2_rev.csv', usecols=['question1', 'question2']).values ret = numpy.array(list(p.map(load_data, df)))
@app.route('/messages') @login_required() def messages_list(): return render_template('messages/messages_list.html', active_nav='messages') @app.route('/messages/<message_id>') @login_required() def message_detail(message_id): return render_template('messages/message_detail.html', active_nav='messages') userClass = user.User(app.config) courseClass = course.Course(app.config) if not app.config['DEBUG']: import logging from logging import FileHandler file_handler = FileHandler(app.config['LOG_FILE']) file_handler.setLevel(logging.WARNING) if __name__ == '__main__': app.run(host="0.0.0.0", port=int(os.environ.get("PORT", 8080)), debug=app.config['DEBUG'], threaded=True)
def create_app(config): app = Flask(__name__) app.config.from_pyfile(config) # Automatically tear down SQLAlchemy. ''' @app.teardown_request def shutdown_session(exception=None): db_session.remove() ''' # Login required decorator. ''' def login_required(test): @wraps(test) def wrap(*args, **kwargs): if 'logged_in' in session: return test(*args, **kwargs) else: flash('You need to login first.') return redirect(url_for('login')) return wrap ''' #----------------------------------------------------------------------------# # Controllers. #----------------------------------------------------------------------------# @app.route('/') def index(): return render_template('pages/index.html') @app.route('/login') def login(): return render_template('pages/login.html') @app.route('/register') def register(): form = RegisterForm(request.form) return render_template('forms/register.html', form=form) @app.route('/forgot') def forgot(): form = ForgotForm(request.form) return render_template('forms/forgot.html', form=form) # Error handlers. @app.errorhandler(500) def internal_error(error): #db_session.rollback() return render_template('errors/500.html'), 500 @app.errorhandler(404) def not_found_error(error): return render_template('errors/404.html'), 404 if not app.debug: file_handler = FileHandler('error.log') file_handler.setFormatter( Formatter('%(asctime)s %(levelname)s: %(message)s [in %(pathname)s:%(lineno)d]') ) app.logger.setLevel(logging.INFO) file_handler.setLevel(logging.INFO) app.logger.addHandler(file_handler) app.logger.info('errors') return app
import warnings warnings.simplefilter('ignore') utils.start(__file__) #============================================================================== # Logger #============================================================================== from logging import getLogger, FileHandler, Formatter, DEBUG logger = getLogger(__name__) logger.setLevel(DEBUG) file_handler = FileHandler(os.path.join('logs', 'log_{}'.format(str(datetime.datetime.today().date()).replace('-', '')))) formatter = Formatter('%(message)s') file_handler.setFormatter(formatter) file_handler.setLevel(DEBUG) logger.addHandler(file_handler) logger.propagate = False #============================================================================== PATH = os.path.join('..', 'data') KEY = 'card_id' SEED = 18 # SEED = np.random.randint(9999) NTHREAD = cpu_count() NFOLD = 11
#!/usr/bin/env python # -*- coding: utf-8 -*- import os from project import app from logging import FileHandler, WARNING file_handler = FileHandler('errorlog.txt') file_handler.setLevel(WARNING) app.logger.addHandler(file_handler) if __name__ == '__main__': port = int(os.environ.get("PORT", 8080)) app.run('0.0.0.0', port=port)
def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--model_name", type=str, help="bert-name used for biencoder") parser.add_argument("--model_path", type=str, help="model save path") parser.add_argument("--index_path", type=str, help="model save path") parser.add_argument("--load_index", action="store_true", help="model save path") parser.add_argument("--mention_dataset", type=str, help="mention dataset path") parser.add_argument("--category", type=str, help="mention dataset path") parser.add_argument("--candidate_dataset", type=str, help="candidate dataset path") parser.add_argument("--candidate_preprocessed", action="store_true", help="whether candidate_dataset is preprocessed") parser.add_argument("--builder_gpu", action="store_true", help="bert-name used for biencoder") parser.add_argument("--max_ctxt_len", type=int, help="maximum context length") parser.add_argument("--max_title_len", type=int, help="maximum title length") parser.add_argument("--max_desc_len", type=int, help="maximum description length") parser.add_argument("--mlflow", action="store_true", help="whether using inbatch negative") parser.add_argument("--parallel", action="store_true", help="whether using inbatch negative") parser.add_argument("--fp16", action="store_true", help="whether using inbatch negative") parser.add_argument('--fp16_opt_level', type=str, default="O1") parser.add_argument("--logging", action="store_true", help="whether using inbatch negative") parser.add_argument("--log_file", type=str, help="whether using inbatch negative") args = parser.parse_args() if args.mlflow: mlflow.start_run() arg_dict = vars(args) for key, value in arg_dict.items(): mlflow.log_param(key, value) logger = None if args.logging: logger = getLogger(__name__) #handler = StreamHandler() logger.setLevel(DEBUG) #handler.setLevel(DEBUG) formatter = Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') #handler.setFormatter(formatter) #logger.addHandler(handler) if args.log_file: fh = FileHandler(filename=args.log_file) fh.setLevel(DEBUG) fh.setFormatter(formatter) logger.addHandler(fh) return args, logger
""" LOG_FORMAT = ( "%(asctime)s \t [%(levelname)s]: \t %(message)s \t in %(pathname)s:%(lineno)d" ) LOG_LEVEL = logging.INFO # canvas logger CANVAS_LOG_FILE = "ACP/logs/canvas.log" canvas_logger = logging.getLogger("Canvas") canvas_logger.setLevel(LOG_LEVEL) canvas_logger_file_handler = FileHandler(CANVAS_LOG_FILE) canvas_logger_file_handler.setLevel(LOG_LEVEL) canvas_logger_file_handler.setFormatter(Formatter(LOG_FORMAT)) canvas_logger.addHandler(canvas_logger_file_handler) # crf logger CRF_LOG_FILE = "ACP/logs/crf.log" crf_logger = logging.getLogger("CRF") crf_logger.setLevel(LOG_LEVEL) crf_file_handler = FileHandler(CRF_LOG_FILE) crf_file_handler.setLevel(LOG_LEVEL) crf_file_handler.setFormatter(Formatter(LOG_FORMAT)) crf_logger.addHandler(crf_file_handler) # email logger
from logging import Logger, FileHandler, Formatter, WARN, INFO DEBUG = False ### logger config logger = Logger('cls') WARN_LOGGER = "/tmp/cls_warn.log" INFO_LOGGER = "/tmp/cls_info.log" formatter = Formatter("%(asctime)s - %(levelname)s - %(message)s") filewarnhandler = FileHandler(WARN_LOGGER, 'a') filewarnhandler.setLevel(WARN) filewarnhandler.setFormatter(formatter) fileinfohandler = FileHandler(INFO_LOGGER, 'a') fileinfohandler.setLevel(INFO) fileinfohandler.setFormatter(formatter) logger.addHandler(filewarnhandler) logger.addHandler(fileinfohandler) # origin_warning = logger.warning # # # def my_warning(*args, **kwargs): # origin_warning(locals()) # return origin_warning(*args, **kwargs) # # # logger.warning = my_warning
with gzip.open(FEATURE_FOLDER + patient_id + '.pkl.gz', 'rb') as f: img = pickle.load(f) img = np.array([rotate(im, -10) for im in img]) #img = rotate(img, -10).astype(np.int8) with gzip.open(FEATURE_FOLDER_OUT + patient_id + '.pkl.gz', 'wb') as f: pickle.dump(img, f, -1) logger.debug('{} img size] {}'.format(patient_id, img.shape)) if __name__ == '__main__': from logging import StreamHandler, DEBUG, Formatter, FileHandler log_fmt = Formatter( '%(asctime)s %(name)s %(lineno)d [%(levelname)s][%(funcName)s] %(message)s ' ) handler = FileHandler('rotate.log', 'w') handler.setLevel(DEBUG) handler.setFormatter(log_fmt) logger.setLevel(DEBUG) logger.addHandler(handler) handler = StreamHandler() handler.setLevel(DEBUG) handler.setFormatter(log_fmt) logger.setLevel(DEBUG) logger.addHandler(handler) load_data()
class ConfigRpmMaker(object): ERROR_MSG = """ ------------------------------------------------------------------------ Your commit has been accepted by the SVN server, but due to the errors that it contains no RPMs have been created. See %s/%s.txt for details. Please fix the issues and trigger the RPM creation with a dummy commit. ------------------------------------------------------------------------ """ def __init__(self, revision, svn_service): self.revision = revision self.svn_service = svn_service self.temp_dir = get_temporary_directory() self._assure_temp_dir_if_set() self._create_logger() self.work_dir = None self.host_queue = Queue() self.failed_host_queue = Queue() def __build_error_msg_and_move_to_public_access(self, revision): err_url = get_error_log_url() error_msg = self.ERROR_MSG % (err_url, revision) for line in error_msg.split('\n'): LOGGER.error(line) self._move_error_log_for_public_access() self._clean_up_work_dir() return error_msg def build(self): LOGGER.info('Working on revision %s', self.revision) self.logger.info("Starting with revision %s", self.revision) try: changed_paths = self.svn_service.get_changed_paths(self.revision) available_hosts = self.svn_service.get_hosts(self.revision) affected_hosts = list( self._get_affected_hosts(changed_paths, available_hosts)) if not affected_hosts: LOGGER.info( "No rpm(s) built. No host affected by change set: %s", str(changed_paths)) return log_elements_of_list(LOGGER.debug, 'Detected %s affected host(s).', affected_hosts) self._prepare_work_dir() rpms = self._build_hosts(affected_hosts) self._upload_rpms(rpms) self._move_configviewer_dirs_to_final_destination(affected_hosts) except BaseConfigRpmMakerException as exception: self.logger.error('Last error during build:\n%s' % str(exception)) self.__build_error_msg_and_move_to_public_access(self.revision) raise exception except Exception as exception: self.logger.exception('Last error during build:') error_msg = self.__build_error_msg_and_move_to_public_access( self.revision) raise Exception( 'Unexpected error occurred, stacktrace will follow.\n%s\n\n%s' % (traceback.format_exc(), error_msg)) self._clean_up_work_dir() return rpms def _clean_up_work_dir(self): if self._keep_work_dir(): LOGGER.info( 'All working data can be found in "{working_directory}"'. format(working_directory=self.work_dir)) else: if self.work_dir and exists(self.work_dir): if is_verbose_enabled(): log_directories_summary(LOGGER.debug, self.work_dir) LOGGER.debug('Cleaning up working directory "%s"', self.work_dir) rmtree(self.work_dir) if exists(self.error_log_file): LOGGER.debug('Removing error log "%s"', self.error_log_file) remove(self.error_log_file) def _keep_work_dir(self): return is_no_clean_up_enabled() def _move_error_log_for_public_access(self): error_log_dir = os.path.join(get_error_log_directory()) if error_log_dir: if not os.path.exists(error_log_dir): os.makedirs(error_log_dir) shutil.move(self.error_log_file, os.path.join(error_log_dir, self.revision + '.txt')) def _read_integer_from_file(self, path): with open(path) as file_which_contains_integer: integer_from_file = int(file_which_contains_integer.read()) return integer_from_file def _move_configviewer_dirs_to_final_destination(self, hosts): LOGGER.info("Updating configviewer data.") for host in hosts: temp_path = build_config_viewer_host_directory( host, revision=self.revision) dest_path = build_config_viewer_host_directory(host) if exists(dest_path): path_to_revision_file = join(dest_path, "%s.rev" % host) revision_from_file = self._read_integer_from_file( path_to_revision_file) if revision_from_file > int(self.revision): LOGGER.debug( 'Will not update configviewer data for host "%s" since the current revision file contains revision %d which is higher than %s', host, revision_from_file, self.revision) rmtree(temp_path) continue rmtree(dest_path) LOGGER.debug('Updating configviewer data for host "%s"', host) move(temp_path, dest_path) def _notify_that_host_failed(self, host_name, stack_trace): failure_information = (host_name, stack_trace) self.failed_host_queue.put(failure_information) approximately_count = self.failed_host_queue.qsize() LOGGER.error( 'Build for host "{host_name}" failed. Approximately {count} builds failed.' .format(host_name=host_name, count=approximately_count)) maximum_allowed_failed_hosts = get_max_failed_hosts() if approximately_count >= maximum_allowed_failed_hosts: LOGGER.error( 'Stopping to build more hosts since the maximum of %d failed hosts has been reached' % maximum_allowed_failed_hosts) self.host_queue.queue.clear() def _build_hosts(self, hosts): if not hosts: LOGGER.warn('Trying to build rpms for hosts, but no hosts given!') return for host in hosts: self.host_queue.put(host) rpm_queue = Queue() svn_service_queue = Queue() svn_service_queue.put(self.svn_service) thread_count = self._get_thread_count(hosts) thread_pool = [ BuildHostThread( name='Thread-%d' % i, revision=self.revision, svn_service_queue=svn_service_queue, rpm_queue=rpm_queue, notify_that_host_failed=self._notify_that_host_failed, host_queue=self.host_queue, work_dir=self.work_dir, error_logging_handler=self.error_handler) for i in range(thread_count) ] for thread in thread_pool: LOGGER.debug('%s: starting ...', thread.name) thread.start() for thread in thread_pool: thread.join() failed_hosts = dict(self._consume_queue(self.failed_host_queue)) if failed_hosts: failed_hosts_str = [ '\n%s:\n\n%s\n\n' % (key, value) for (key, value) in failed_hosts.iteritems() ] raise CouldNotBuildSomeRpmsException( "Could not build config rpm for some host(s): %s" % '\n'.join(failed_hosts_str)) LOGGER.info("Finished building configuration rpm(s).") built_rpms = self._consume_queue(rpm_queue) log_elements_of_list(LOGGER.debug, 'Built %s rpm(s).', built_rpms) return built_rpms @measure_execution_time def _upload_rpms(self, rpms): rpm_upload_cmd = get_rpm_upload_command() chunk_size = self._get_chunk_size(rpms) if rpm_upload_cmd: LOGGER.info("Uploading %s rpm(s).", len(rpms)) LOGGER.debug( 'Uploading rpm(s) using command "%s" and chunk_size "%s"', rpm_upload_cmd, chunk_size) pos = 0 while pos < len(rpms): rpm_chunk = rpms[pos:pos + chunk_size] cmd = '%s %s' % (rpm_upload_cmd, ' '.join(rpm_chunk)) process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() if process.returncode: error_message = 'Rpm upload failed with exit code %s. Executed command "%s"\n' % ( process.returncode, cmd) if stdout: error_message += 'stdout: "%s"\n' % stdout.strip() if stderr: error_message += 'stderr: "%s"\n' % stderr.strip() raise CouldNotUploadRpmsException(error_message) pos += chunk_size else: LOGGER.info( "Rpms will not be uploaded since no upload command has been configured." ) def _get_affected_hosts(self, changed_paths, available_host): result = set() for segment in OVERLAY_ORDER: for changed_path in changed_paths: result |= set( self._find_matching_hosts(segment, changed_path, available_host)) return result def _find_matching_hosts(self, segment, svn_path, available_hosts): result = [] for host in available_hosts: for path in segment.get_svn_paths(host): if svn_path.startswith(path): result.append(host) break return result def _get_thread_count(self, affected_hosts): thread_count = int(get_thread_count()) if thread_count < 0: raise ConfigurationException( '%s is %s, values <0 are not allowed)' % (get_thread_count, thread_count)) if not thread_count or thread_count > len(affected_hosts): if not thread_count: reason = 'Configuration property "%s" is %s' % ( get_thread_count, thread_count) elif thread_count > len(affected_hosts): reason = "More threads available than affected hosts" thread_count = len(affected_hosts) LOGGER.info("%s: using one thread for each affected host." % (reason)) return thread_count def _consume_queue(self, queue): items = [] while not queue.empty(): item = queue.get() queue.task_done() items.append(item) return items def _create_logger(self): self.error_log_file = tempfile.mktemp(dir=get_temporary_directory(), prefix='yadt-config-rpm-maker.', suffix='.revision-%s.error.log' % self.revision) self.error_handler = FileHandler(self.error_log_file) formatter = Formatter(configuration.LOG_FILE_FORMAT, configuration.LOG_FILE_DATE_FORMAT) self.error_handler.setFormatter(formatter) self.error_handler.setLevel(ERROR) self.logger = getLogger('fileLogger') self.logger.addHandler(self.error_handler) self.logger.propagate = False def _assure_temp_dir_if_set(self): if self.temp_dir and not os.path.exists(self.temp_dir): os.makedirs(self.temp_dir) def _prepare_work_dir(self): LOGGER.debug('Preparing working directory "%s"', self.temp_dir) self.work_dir = mkdtemp(prefix='yadt-config-rpm-maker.', suffix='.' + self.revision, dir=self.temp_dir) self.rpm_build_dir = join(self.work_dir, 'rpmbuild') LOGGER.debug('Creating directory structure for rpmbuild in "%s"', self.rpm_build_dir) for name in [ 'tmp', 'RPMS', 'RPMS/x86_64', 'RPMS/noarch', 'BUILD', 'BUILDROOT', 'SRPMS', 'SPECS', 'SOURCES' ]: path = join(self.rpm_build_dir, name) if not exists(path): makedirs(path) def _get_chunk_size(self, rpms): chunk_size_raw = get_rpm_upload_chunk_size() try: chunk_size = int(chunk_size_raw) except ValueError: raise ConfigurationException( 'rpm_upload_chunk_size (%s) is not a legal value (should be int)' % chunk_size_raw) if chunk_size < 0: raise ConfigurationException( "Config param 'rpm_upload_cmd_chunk_size' needs to be greater or equal 0" ) if not chunk_size: chunk_size = len(rpms) return chunk_size
from flask import Flask, request, Response import os from copr_keygen.exceptions import BadRequestException, \ KeygenServiceBaseException app = Flask(__name__) app.config.from_object("copr_keygen.default_settings") app.config.from_envvar("COPR_KEYGEN_CONFIG", silent=True) # setup logger if not app.config["DEBUG"] or app.config["DEBUG_WITH_LOG"]: filename = os.path.join(app.config["LOG_DIR"], "main.log") if os.path.exists(app.config["LOG_DIR"]): handler = FileHandler(filename) handler.setLevel(app.config["LOG_LEVEL"]) handler.setFormatter(logging.Formatter( '%(asctime)s %(levelname)s' '[%(module)s:%(pathname)s:%(lineno)d]' ': %(message)s ' )) logger = getLogger(__name__) logger.addHandler(handler) logger.setLevel(app.config["LOG_LEVEL"]) # end setup logger from .logic import create_new_key, user_exists log = logging.getLogger(__name__)
def setUpClass(cls): """ Perform class setup before running the testcase Remove shared memory files, start vpp and connect the vpp-api """ gc.collect() # run garbage collection first cls.logger = getLogger(cls.__name__) cls.tempdir = tempfile.mkdtemp(prefix='vpp-unittest-' + cls.__name__ + '-') file_handler = FileHandler("%s/log.txt" % cls.tempdir) file_handler.setFormatter( Formatter(fmt='%(asctime)s,%(msecs)03d %(message)s', datefmt="%H:%M:%S")) file_handler.setLevel(DEBUG) cls.logger.addHandler(file_handler) cls.shm_prefix = cls.tempdir.split("/")[-1] os.chdir(cls.tempdir) cls.logger.info("Temporary dir is %s, shm prefix is %s", cls.tempdir, cls.shm_prefix) cls.setUpConstants() cls.reset_packet_infos() cls._captures = [] cls._zombie_captures = [] cls.verbose = 0 cls.vpp_dead = False cls.registry = VppObjectRegistry() # need to catch exceptions here because if we raise, then the cleanup # doesn't get called and we might end with a zombie vpp try: cls.run_vpp() cls.vpp_stdout_deque = deque() cls.vpp_stderr_deque = deque() cls.pump_thread_stop_flag = Event() cls.pump_thread_wakeup_pipe = os.pipe() cls.pump_thread = Thread(target=pump_output, args=(cls, )) cls.pump_thread.daemon = True cls.pump_thread.start() cls.vapi = VppPapiProvider(cls.shm_prefix, cls.shm_prefix, cls) if cls.step: hook = StepHook(cls) else: hook = PollHook(cls) cls.vapi.register_hook(hook) cls.sleep(0.1, "after vpp startup, before initial poll") hook.poll_vpp() try: cls.vapi.connect() except: if cls.debug_gdbserver: print( colorize( "You're running VPP inside gdbserver but " "VPP-API connection failed, did you forget " "to 'continue' VPP from within gdb?", RED)) raise except: t, v, tb = sys.exc_info() try: cls.quit() except: pass raise t, v, tb
class MainBase(wx.App): def __init__(self): """アプリを初期化する。""" super().__init__() #実行環境の取得(exeファイルorインタプリタ) self.frozen=hasattr(sys,"frozen") #各種初期設定 self.InitLogger() self.LoadSettings() try: if self.config["general"]["locale"]!=None: locale.setlocale(locale.LC_TIME,self.config["general"]["locale"]) else: locale.setlocale(locale.LC_TIME) except: locale.setlocale(locale.LC_TIME) self.config["general"]["locale"]="" self.SetTimeZone() self.InitTranslation() self.InitSpeech() # ログのファイルハンドラーが利用可能でなければ警告を出す if not self.log.hasHandlers(): simpleDialog.errorDialog(_("ログ機能の初期化に失敗しました。下記のファイルへのアクセスが可能であることを確認してください。") + "\n" + os.path.abspath(constants.LOG_FILE_NAME)) def InitSpeech(self): # 音声読み上げの準備 try: self._InitSpeech() except OutputError as e: self.log.error("Failed to initialize speech output.") self.log.error(traceback.format_exc()) simpleDialog.winDialog(_("音声エンジンエラー"), _("音声読み上げ機能の初期化に失敗したため、読み上げ機能を使用できません。出力先の変更をお試しください。")) self.speech = accessible_output2.outputs.nospeech.NoSpeech() def _InitSpeech(self): # 音声読み上げの準備 reader=self.config["speech"]["reader"] if(reader=="PCTK"): self.log.info("use reader 'PCTalker'") self.speech=accessible_output2.outputs.pc_talker.PCTalker() elif(reader=="NVDA"): self.log.info("use reader 'NVDA'") self.speech=accessible_output2.outputs.nvda.NVDA() #SAPI4はバグってるっぽいので無効にしておく # elif(reader=="SAPI4"): # self.log.info("use reader 'SAPI4'") # self.speech=accessible_output2.outputs.sapi4.Sapi4() elif(reader=="SAPI5"): self.log.info("use reader 'SAPI5'") self.speech=accessible_output2.outputs.sapi5.SAPI5() elif(reader=="AUTO"): self.log.info("use reader 'AUTO'") self.speech=accessible_output2.outputs.auto.Auto() elif(reader=="JAWS"): self.log.info("use reader 'JAWS'") self.speech=accessible_output2.outputs.jaws.Jaws() elif(reader=="CLIPBOARD"): self.log.info("use reader 'CLIPBOARD'") self.speech=accessible_output2.outputs.clipboard.Clipboard() elif(reader=="NOSPEECH"): self.log.info("use reader 'NOSPEECH'") self.speech=accessible_output2.outputs.nospeech.NoSpeech() else: self.config.set("speech","reader","AUTO") self.log.warning("Setting missed! speech.reader reset to 'AUTO'") self.speech=accessible_output2.outputs.auto.Auto() def InitLogger(self): """ログ機能を初期化して準備する。""" try: self.hLogHandler=FileHandler(constants.LOG_FILE_NAME, mode="w", encoding="UTF-8") self.hLogHandler.setLevel(logging.DEBUG) self.hLogFormatter=Formatter("%(name)s - %(levelname)s - %(message)s (%(asctime)s)") self.hLogHandler.setFormatter(self.hLogFormatter) logger=getLogger(constants.LOG_PREFIX) logger.setLevel(logging.DEBUG) logger.addHandler(self.hLogHandler) except Exception as e: traceback.print_exc() self.log=getLogger(constants.LOG_PREFIX+".Main") r="executable" if self.frozen else "interpreter" self.log.info("Starting"+constants.APP_NAME+" "+constants.APP_VERSION+" as %s!" % r) def LoadSettings(self): """設定ファイルを読み込む。なければデフォルト設定を適用し、設定ファイルを書く。""" self.config = DefaultSettings.DefaultSettings.get() if not self.config.read(constants.SETTING_FILE_NAME): #初回起動 self.config.read_dict(DefaultSettings.initialValues) self.config.write() self.hLogHandler.setLevel(self.config.getint("general","log_level",20,0,50)) def InitTranslation(self): """翻訳を初期化する。""" loc = locale.getdefaultlocale()[0].replace("_", "-") lang=self.config.getstring("general","language","",constants.SUPPORTING_LANGUAGE.keys()) if lang == "": if loc in list(constants.SUPPORTING_LANGUAGE.keys()): self.config["general"]["language"] = loc else: # 言語選択を表示 langSelect = views.langDialog.langDialog() langSelect.Initialize() langSelect.Show() self.config["general"]["language"] = langSelect.GetValue() lang = self.config["general"]["language"] self.translator=gettext.translation("messages","locale", languages=[lang], fallback=True) self.translator.install() def GetFrozenStatus(self): """コンパイル済みのexeで実行されている場合はTrue、インタプリタで実行されている場合はFalseを帰す。""" return self.frozen def say(self,s,interrupt=False): """スクリーンリーダーでしゃべらせる。""" self.speech.speak(s, interrupt=interrupt) self.speech.braille(s) def SetTimeZone(self): bias=win32api.GetTimeZoneInformation(True)[1][0]*-1 hours=bias//60 minutes=bias%60 self.timezone=datetime.timezone(datetime.timedelta(hours=hours,minutes=minutes)) def getAppPath(self): """アプリの絶対パスを返す """ if self.frozen: # exeファイルで実行されている return sys.executable else: # pyファイルで実行されている return os.path.join(os.path.dirname(__file__), os.path.basename(sys.argv[0]))
from flask_restful import Api from flask_sqlalchemy import SQLAlchemy from .config import test_config from .config import app_config app = Flask("Entity matching tool") # app.config.from_object(test_config) app.config.from_object(app_config) db = SQLAlchemy(app) api = Api(app) handler = FileHandler('log_file') formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) handler.setLevel(logging.INFO) app.logger.addHandler(handler) # try: # mongo = connect(test_config.MONGO['db']) # conn = psycopg2.connect("dbname='{}' user='******' " # "host='{}' password='******'".format(test_config.POSTGRES['db'], # test_config.POSTGRES['user'], # test_config.POSTGRES['host'], # test_config.POSTGRES['pw'])) # except psycopg2.OperationalError as e: # engine = sqlalchemy.create_engine("postgres://{}:{}@{}/{}".format(test_config.POSTGRES['user'], # test_config.POSTGRES['pw'], # test_config.POSTGRES['host'], # test_config.POSTGRES['db'])) # if not database_exists(engine.url):
LOG_FORMAT = ( "%(asctime)s [%(levelname)s]: %(message)s in %(pathname)s:%(lineno)d") LOG_LEVEL = logging.INFO # messaging logger MESSAGING_LOG_FILE = "project.log" logging.basicConfig(format='%(asctime)s | %(levelname)s : %(message)s', level=logging.INFO, stream=sys.stdout) quest_gen_logger = logging.getLogger("project.quest_gen") quest_gen_logger.setLevel(LOG_LEVEL) quest_gen_logger_file_handler = FileHandler(MESSAGING_LOG_FILE) quest_gen_logger_file_handler.setLevel(LOG_LEVEL) quest_gen_logger_file_handler.setFormatter(Formatter(LOG_FORMAT)) quest_gen_logger.addHandler(quest_gen_logger_file_handler) class RandomGenerator: """ Random generator controlling the games generation. """ def __init__(self, seed=None): self.set_seed(seed) @property def seed(self): return self._seed def set_seed(self, seed): self._seed = seed
# -*- coding: utf-8 -*- import os import json import paho.mqtt.client as mqtt import camera_mode_selector from logging import getLogger, FileHandler, StreamHandler, DEBUG logger = getLogger(__name__) if not logger.handlers: fileHandler = FileHandler(r'./log/camera_runner.log') fileHandler.setLevel(DEBUG) streamHander = StreamHandler() streamHander.setLevel(DEBUG) logger.setLevel(DEBUG) logger.addHandler(fileHandler) logger.addHandler(streamHander) # MQTT broker server host = os.getenv('SSS_MQTT_HOST') port = int(os.getenv('SSS_MQTT_PORT')) # subscribe topic sub_topic = 'sensor/event' # publish topic pub_topic = 'sensor/feedback/result/' def on_connect(client, data, flags, response_code): logger.info('status {0}'.format(response_code)) camera_mode_selector.change_mode('2')
import os import json import time import datetime import traceback import dropbox # from logging import getLogger, StreamHandler, FileHandler, DEBUG, INFO, WARN, ERROR logger = getLogger(__name__) sh = StreamHandler() fh = FileHandler("/pws/log/uploader.log") sh.setLevel(INFO) fh.setLevel(INFO) logger.setLevel(INFO) logger.addHandler(sh) logger.addHandler(fh) # def get_log_header(): return "{0} {1}".format(datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S"), os.path.splitext(os.path.basename(__file__))[0]) # class DropboxConfig(object): APP_KEY_NAME = "app_key" APP_SECRET_NAME = "app_secret" ACCESS_TOKEN_NAME = "access_token" #
# encoding: utf-8 from webapp import app from logging import FileHandler, INFO, Formatter from config import current_config handler = FileHandler(current_config.flask_log, encoding='UTF-8') logging_format = Formatter( '%(levelname)s %(asctime)s [%(name)s:%(module)s:%(funcName)s:%(lineno)s] %(message)s' ) handler.setFormatter(logging_format) handler.setLevel(INFO) app.logger.addHandler(handler)
def main(): parser = argparse.ArgumentParser(description='argparse example.') parser.add_argument('input', help="Root directory", metavar='<input>') parser.add_argument('output', help="Output directory", metavar='<output>') parser.add_argument('--ext', help='File extension. default: %(default)s', metavar='str', default='.mha') parser.add_argument( '--prefix', help='Prefix of the output filename. default: %(default)s', metavar='str', default='SE') parser.add_argument('--compress', help='Compress the output image. default: %(default)s', type=str, choices=['auto', 'true', 'false'], default='auto') parser.add_argument('--offset', help='Offset to the number. default: %(default)s', type=int, metavar='int', default=1) parser.add_argument('--logdir', help='Directory to store logs. default: %(default)s', metavar='str', default=None) parser.add_argument('--verbose', help='Verbosity. default: %(default)s', type=int, metavar='level', default=0) args = parser.parse_args() logger.setLevel(verbosity_to_level(args.verbose)) if args.logdir is not None: logdir = Path(args.logdir) logdir.mkdir(parents=True, exist_ok=True) handler = FileHandler( logdir / '{}.log'.format(datetime.today().strftime("%y%m%d_%H%M%S"))) handler.setLevel(verbosity_to_level(args.verbose)) handler.setFormatter(log_format) logger.addHandler(handler) root_dir = Path(args.input) out_dir = Path(args.output) compression = {'auto': None, 'true': True, 'false': False}[args.compress] dtype = None prefix = args.prefix ext = args.ext offset = args.offset logger.info('Collect dicom information') all_files = [ str(e) for e in tqdm.tqdm(root_dir.glob('**/*'), desc='list all files') if e.is_file() ] key_tags = [ 'PatientID', 'SeriesInstanceUID', 'AcquisitionDate', 'AcquisitionTime', 'ImageOrientationPatient', 'ImagePositionPatient' ] dcm_files = [] for fn in tqdm.tqdm(all_files): try: dcm = pydicom.dcmread(fn, stop_before_pixels=True) dcm_files.append([fn] + [dcm.get(tag) for tag in key_tags]) except Exception as e: logger.warning({'filename': fn, 'exception': e}) df = pd.DataFrame(dcm_files, columns=['filepath'] + key_tags) logger.info('Convert dicom files') def sort_dicom(df): orientation = np.array(df['ImageOrientationPatient'].iloc[0]).reshape( (2, 3)) third_axis = np.cross(orientation[0], orientation[1]) locs = df['ImagePositionPatient'].map(lambda p: np.dot(third_axis, p)) sorted_index = np.argsort(locs) return df.iloc[sorted_index] FLOAT_TYPES = set([ sitk.sitkFloat32, sitk.sitkFloat64, sitk.sitkVectorFloat32, sitk.sitkVectorFloat64 ]) for patient_id, df_patient in df.groupby('PatientID'): logger.info(patient_id) sids, times = [], [] for series_id, df_series in df_patient.groupby('SeriesInstanceUID'): sids.append(series_id) dts = df_series.apply( lambda row: DT(row.AcquisitionDate + row.AcquisitionTime), axis=1).tolist() if len(df_series) <= 2: times.append(dts[0]) else: dts.sort() times.append(dts[len(dts) // 2]) nums = np.argsort(np.argsort(times)) series_id2series_number = dict(zip(sids, nums)) for series_id, df_series in df_patient.groupby('SeriesInstanceUID'): logger.debug(series_id) output_filename = out_dir / patient_id / (prefix + '{:d}'.format( series_id2series_number[series_id] + offset) + ext) output_filename.parent.mkdir(parents=True, exist_ok=True) filenames = sort_dicom(df_series)['filepath'].tolist() reader = sitk.ImageSeriesReader() reader.SetFileNames(filenames) image = reader.Execute() if image.GetPixelID() == sitk.sitkFloat64 and dtype is None: f = sitk.CastImageFilter() f.SetOutputPixelType(sitk.sitkFloat32) image = f.Execute(image) writer = sitk.ImageFileWriter() if compression is None: compression = image.GetPixelID() not in FLOAT_TYPES writer.SetUseCompression(compression) writer.SetFileName(str(output_filename)) writer.Execute(image) logger.info('End')
def start(resultdir: str, configfile: str): basepath = Path(resultdir) print(("storing results in {}".format(basepath))) now = datetime.now() timestamp = now.strftime("%Y_%m_%d-%H_%M_%S") experiment_path = basepath.joinpath(timestamp + "_" + STRATEGY) num = 2 while experiment_path.exists(): experiment_path = basepath.joinpath(timestamp + "_" + "v" + str(num)) experiment_path.mkdir() MODULE_LOGGER_NAME = 'salma' # logging.config.fileConfig("experiment01.logging.conf") logging.basicConfig() logger = logging.getLogger(MODULE_LOGGER_NAME) logger.setLevel(logging.DEBUG) fh = FileHandler(str(experiment_path / "experiment.log")) fh.setLevel(logging.DEBUG) logger.addHandler(fh) experiment = Experiment01(experiment_path, None if configfile is None else Path(configfile)) experiment.initialize() runner = SingleProcessExperimentRunner() with experiment_path.joinpath("experiment.csv").open("w") as f: f.write(create_csv_header() + "\n") f.flush() experiment.step_listeners.append(create_step_logger(f)) experiment.step_listeners.append(break_when_all_delivered) experiment.step_listeners.append(break_when_all_broken) # _, res, trial_infos = runner.run_trials(experiment, number_of_trials=1, max_steps=3000, max_retrials=0) experiment.run(max_steps=5000) experiment.world.printState()