def __init__(self, config_file, gpu='-1', debug=False): """Load parameters and set log level. Args: config_file (str): path to the config file, which should be in ``yaml`` format. You can use default config provided in the `Github repo`_, or write it by yourself. debug (bool, optional): whether to enable debug function during running. Defaults to False. .. _Github repo: https://github.com/RUCAIBox/CRSLab """ self.opt = self.load_yaml_configs(config_file) # gpu self.opt['gpu'] = gpu # dataset dataset = self.opt['dataset'] tokenize = self.opt['tokenize'] if isinstance(tokenize, dict): tokenize = ', '.join(tokenize.values()) # model model = self.opt.get('model', None) rec_model = self.opt.get('rec_model', None) conv_model = self.opt.get('conv_model', None) policy_model = self.opt.get('policy_model', None) if model: model_name = model else: models = [] if rec_model: models.append(rec_model) if conv_model: models.append(conv_model) if policy_model: models.append(policy_model) model_name = '_'.join(models) self.opt['model_name'] = model_name # log log_name = self.opt.get( "log_name", dataset + '_' + model_name + '_' + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())) + ".log" if not os.path.exists("log"): os.makedirs("log") logger.remove() if debug: level = 'DEBUG' else: level = 'INFO' logger.add(os.path.join("log", log_name), level=level) logger.add(lambda msg: tqdm.write(msg, end=''), colorize=True, level=level) logger.info(f"[Dataset: {dataset} tokenized in {tokenize}]") if model: logger.info(f'[Model: {model}]') if rec_model: logger.info(f'[Recommendation Model: {rec_model}]') if conv_model: logger.info(f'[Conversation Model: {conv_model}]') if policy_model: logger.info(f'[Policy Model: {policy_model}]') logger.info("[Config]" + '\n' + json.dumps(self.opt, indent=4))
def run(args): hosts = args.node or DEFAULT_NODES if not args.verbose: LOG.remove() LOG.add( sys.stdout, format="<green>[{time:HH:mm:ss.SSS}]</green> {message}", ) LOG.disable("infra") LOG.disable("ccf") LOG.info( f"Starting {len(hosts)} CCF node{'s' if len(hosts) > 1 else ''}...") if args.enclave_type == "virtual": LOG.warning("Virtual mode enabled") with infra.network.network( hosts=hosts, binary_directory=args.binary_dir, library_directory=args.library_dir, dbg_nodes=args.debug_nodes, ) as network: if args.recover: args.label = args.label + "_recover" LOG.info("Recovering network from:") LOG.info(f" - Common directory: {args.common_dir}") LOG.info(f" - Ledger: {args.ledger_dir}") if args.snapshot_dir: LOG.info(f" - Snapshots: {args.snapshot_dir}") else: LOG.warning( "No available snapshot to recover from. Entire transaction history will be replayed." ) network.start_in_recovery( args, args.ledger_dir, snapshot_dir=args.snapshot_dir, common_dir=args.common_dir, ) network.recover(args) else: network.start_and_join(args) nodes = network.get_joined_nodes() max_len = max([len(str(node.local_node_id)) for node in nodes]) # To be sure, confirm that the app frontend is open on each node for node in nodes: with node.client("user0") as c: if args.verbose: r = c.get("/app/commit") else: r = c.get("/app/commit", log_capture=[]) assert r.status_code == http.HTTPStatus.OK, r.status_code def pad_node_id(nid): return (f"{{:{max_len}d}}").format(nid) LOG.info("Started CCF network with the following nodes:") for node in nodes: LOG.info(" Node [{}] = https://{}:{}".format( pad_node_id(node.local_node_id), node.pubhost, node.pubport)) LOG.info( f"You can now issue business transactions to the {args.package} application" ) if args.js_app_bundle is not None: LOG.info(f"Loaded JS application: {args.js_app_bundle}") LOG.info( f"Keys and certificates have been copied to the common folder: {network.common_dir}" ) LOG.info( "See https://microsoft.github.io/CCF/main/use_apps/issue_commands.html for more information" ) LOG.warning("Press Ctrl+C to shutdown the network") try: while True: time.sleep(60) except KeyboardInterrupt: LOG.info("Stopping all CCF nodes...") LOG.info("All CCF nodes stopped.")
def Main(): default_config = os.path.join(CCF_Etc, 'cchost.toml') default_output = os.path.join(CCF_Keys, 'ledger_authority.pem') parser = argparse.ArgumentParser( description='Fetch the ledger authority key from a CCF server') parser.add_argument( '--logfile', help='Name of the log file, __screen__ for standard output', default='__screen__', type=str) parser.add_argument('--loglevel', help='Logging level', default='WARNING', type=str) parser.add_argument('--ccf-config', help='Name of the CCF configuration file', default=default_config, type=str) parser.add_argument('--user-name', help="Name of the user being added", default="userccf", type=str) options = parser.parse_args() # ----------------------------------------------------------------- LOG.remove() if options.logfile == '__screen__': LOG.add(sys.stderr, level=options.loglevel) else: LOG.add(options.logfile) # ----------------------------------------------------------------- try: config = toml.load(options.ccf_config) except: LOG.error('unable to load ccf configuration file {0}'.format( options.ccf_config)) pass network_cert = config["start"]["network-cert-file"] (host, port) = config["rpc-address"].split(':') user_cert_file = os.path.join(CCF_Keys, "{}_cert.pem".format(options.user_name)) user_key_file = os.path.join(CCF_Keys, "{}_privk.pem".format(options.user_name)) try: user_client = CCFClient(host=host, port=port, cert=user_cert_file, key=user_key_file, ca=network_cert, format='json', prefix='app', description="none", version="2.0", connection_timeout=3, request_timeout=3) except: LOG.error('failed to connect to CCF service') sys.exit(-1) #Temporary fix to skip checking CCF host certificate. Version 0.11.7 CCF certificate expiration was hardcoded to end of 2021 user_client.client_impl.session.mount("https://", HTTPAdapter()) user_client.client_impl.session.verify = False generate_ledger_authority(user_client, options, config) LOG.info('successfully generated ledger authority') sys.exit(0)
def thread_2(): barrier.wait() time.sleep(0.5) logger.remove(a) logger.info("ccc{}ddd", next(counter))
# 终端日志输出格式 stdout_fmt = '<cyan>{time:HH:mm:ss,SSS}</cyan> ' \ '[<level>{level: <5}</level>] ' \ '<blue>{module}</blue>:<cyan>{line}</cyan> - ' \ '<level>{message}</level>' # 日志文件记录格式 logfile_fmt = '<light-green>{time:YYYY-MM-DD HH:mm:ss,SSS}</light-green> ' \ '[<level>{level: <5}</level>] ' \ '<cyan>{process.name}({process.id})</cyan>:' \ '<cyan>{thread.name: <10}({thread.id: <5})</cyan> | ' \ '<blue>{module}</blue>.<blue>{function}</blue>:' \ '<blue>{line}</blue> - <level>{message}</level>' log_path = result_save_path.joinpath('oneforall.log') logger.remove() logger.level(name='TRACE', no=5, color='<cyan><bold>', icon='✏️') logger.level(name='DEBUG', no=10, color='<blue><bold>', icon='🐞 ') logger.level(name='INFOR', no=20, color='<green><bold>', icon='ℹ️') logger.level(name='ALERT', no=30, color='<yellow><bold>', icon='⚠️') logger.level(name='ERROR', no=40, color='<red><bold>', icon='❌️') logger.level(name='FATAL', no=50, color='<RED><bold>', icon='☠️') if not os.environ.get('PYTHONIOENCODING'): # 设置编码 os.environ['PYTHONIOENCODING'] = 'utf-8' logger.add(sys.stderr, level='INFOR', format=stdout_fmt, enqueue=True) logger.add(log_path, level='DEBUG', format=logfile_fmt, enqueue=True,
def common_setup(_C: Config, _A: argparse.Namespace, job_type: str = "pretrain"): r""" Setup common stuff at the start of every pretraining or downstream evaluation job, all listed here to avoid code duplication. Basic steps: 1. Fix random seeds and other PyTorch flags. 2. Set up a serialization directory and loggers. 3. Log important stuff such as config, process info (useful during distributed training). 4. Save a copy of config to serialization directory. .. note:: It is assumed that multiple processes for distributed training have already been launched from outside. Functions from :mod:`virtex.utils.distributed` module ae used to get process info. Parameters ---------- _C: virtex.config.Config Config object with all the parameters. _A: argparse.Namespace Command line arguments. job_type: str, optional (default = "pretrain") Type of job for which setup is to be done; one of ``{"pretrain", "downstream"}``. """ # Get process rank and world size (assuming distributed is initialized). RANK = dist.get_rank() WORLD_SIZE = dist.get_world_size() # For reproducibility - refer https://pytorch.org/docs/stable/notes/randomness.html torch.manual_seed(_C.RANDOM_SEED) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False random.seed(_C.RANDOM_SEED) np.random.seed(_C.RANDOM_SEED) # Create serialization directory and save config in it. os.makedirs(_A.serialization_dir, exist_ok=True) _C.dump(os.path.join(_A.serialization_dir, f"{job_type}_config.yaml")) # Remove default logger, create a logger for each process which writes to a # separate log-file. This makes changes in global scope. logger.remove(0) if dist.get_world_size() > 1: logger.add( os.path.join(_A.serialization_dir, f"log-rank{RANK}.txt"), format="{time} {level} {message}", ) # Add a logger for stdout only for the master process. if dist.is_master_process(): logger.add(sys.stdout, format="<g>{time}</g>: <lvl>{message}</lvl>", colorize=True) # Print process info, config and args. logger.info(f"Rank of current process: {RANK}. World size: {WORLD_SIZE}") logger.info(str(_C)) logger.info("Command line args:") for arg in vars(_A): logger.info("{:<20}: {}".format(arg, getattr(_A, arg)))
def remove(): barrier.wait() time.sleep(0.5) logger.remove(j)
def _setup_logger(): logger.remove(0) logger.add(sys.stderr, level=config.Logging.level)
#!/usr/bin/env python -W ignore::FutureWarning -W ignore::UserWarning -W ignore::DeprecationWarning """PINT-based tool for making simulated TOAs.""" import astropy.units as u import numpy as np from astropy.time import TimeDelta import sys import pint.logging from loguru import logger as log log.remove() log.add( sys.stderr, level="WARNING", colorize=True, format=pint.logging.format, filter=pint.logging.LogFilter(), ) import pint.fitter import pint.models import pint.toa as toa import pint.simulation import pint.residuals __all__ = ["main"] def main(argv=None): import argparse
def main(): #print(sys.argv[1]) start_time = time() file_counter = 0 nucleotides = ["A", "T", "G", "C"] if sys.argv[1].strip() == "count_first_in_duplex" or \ sys.argv[1].strip() == "count_second_in_duplex": nuc_pos_in_duplex_to_count = sys.argv[1].strip() #print(nuc_pos_in_duplex_to_count) ############################# # logging if not os.path.exists("./log"): os.mkdir("./log") logger.remove() # don't put messages into notebook output logger.add("./log/apobec_count_snp_duplex_log_{time}" + "_" + nuc_pos_in_duplex_to_count + ".txt", backtrace=False) logger.add(sys.stderr, level="CRITICAL") ############################################# if os.path.exists("./input_data"): #TODO make if not and then return # to get rid of else block below input_files = get_input_files_names("./input_data") num_files = len(input_files) print(""" --------------- job started at {0} ... --------------- """.format(get_current_time())) progress_bar = IntProgress(min=0, max=num_files, bar_style='success') display(progress_bar) for f in input_files: try: df_raw_snp_container = [] df_duplex_container = [] ref_seq = get_ref(f) #################################################### # nuc we use in loop is very important # 1. it's the nuc we use to find all duplexes in reference # which start from it # 2. then we search in count_snp func to which nucs it changes # in the same position in the read and in the same context as in the read # (i.e. the second nuc in duplex is the same as in reference but the fitst nuc ( # which is "nuc" var in the for loop below) changes) for nuc in nucleotides: snp_type = nucleotides[:] # remove nuc from snp type snp_type.remove(nuc) if nuc_pos_in_duplex_to_count == "count_first_in_duplex": duplex_posits = get_duplex_posits_with_ref_nuc_at_first_pos(ref_seq, nuc) df = create_df(duplex_posits, snp_type) df_snp, coverage, record_id = count_snp_with_ref_nuc_at_first_pos(f, df, nuc) #print(df_snp) elif nuc_pos_in_duplex_to_count == "count_second_in_duplex": duplex_posits = get_duplex_posits_with_ref_nuc_at_second_pos(ref_seq, nuc) df = create_df(duplex_posits, snp_type) df_snp, coverage, record_id = count_snp_with_ref_nuc_at_second_pos(f, df, nuc) #print(df_snp) df_duplex_in_context = create_pivot_df(df_snp) #print(df_duplex_in_context) #print("index [0][0]", df_duplex_in_context.index[0][0]) #print("index [0][1]", df_duplex_in_context.index[0][1]) df_raw_snp_container.append(df_snp) df_duplex_container.append(df_duplex_in_context) df_perc_container = convert_pivot_df_into_percent(df_duplex_container) df_cov = pd.DataFrame.from_dict( {"coverage": coverage}, orient='index') # two function with condition, so as not to have conditions inside a single function if nuc_pos_in_duplex_to_count == "count_first_in_duplex": save_df_first_in_duplex(f, df_raw_snp_container, df_duplex_container, df_perc_container, df_cov, nuc_pos_in_duplex_to_count) elif nuc_pos_in_duplex_to_count == "count_second_in_duplex": save_df_second_in_duplex(f, df_raw_snp_container, df_duplex_container, df_perc_container, df_cov, nuc_pos_in_duplex_to_count) progress_bar.value += 1 file_counter += 1 except Exception: logger.info( """\n\t================================================== file: {0} sequence id: {1} ====================================================""".format(f, record_id)) logger.exception("") print("exception detected. see log for more details") progress_bar.value += 1 finish_time = time() total_time = finish_time - start_time show_report(total_time, file_counter) else: os.mkdir("./input_data") print( """ Houston, we have a problem... -------- folder 'input_data' doesn't exist in the current directory or may be you've created it but misspelled its name. Anyway, it has just been created by this script. Paste your 'fasta' files into the folder 'input_data' and run this script again. -------- """ )
from pathlib import Path from typing import * import pandas from loguru import logger from statsmodels.sandbox.stats.multicomp import TukeyHSDResults # Used to add a typing annotation to tukeyhsd() import analysis import projectoutput import utilities from analysis import growthcurver from projectpaths import Filenames TRACE = True if TRACE: logger.remove( ) # Need to remove the default sink so that the logger doesn't print messages twice. import sys logger.add(sys.stderr, level="TRACE") pandas.set_option('mode.chained_assignment', None) EXPECTED_FORMAT = "[strain].[consition].[plate].[replicate]" def extract_labels_from_metadata(metadata: pandas.DataFrame, column: str, allowed_strains: List[str] = None, allowed_conditions: List[str] = None) -> \ List[str]: """ Extracts sample ids based on a specific column in the sample metadata table. Parameters ---------- metadata: pandas.DataFrame
def caplog(caplog): handler_id = logger.add(caplog.handler, format="{message}") yield caplog logger.remove(handler_id)
def base_logger(): """Initialize logging instance.""" _loguru_logger.remove() _loguru_logger.add(sys.stdout, format=_LOG_FMT, level="INFO", enqueue=True) _loguru_logger.configure(levels=_LOG_LEVELS) return _loguru_logger
def main_entry_point(argv=None): arguments = docopt(__doc__, version=__version__, argv=argv) logger.remove(0) # Don't log directly to stderr if arguments["--debug"]: log_level = "DEBUG" log_fmt = ( "[decofre] " "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <8}</level> |" "<cyan>{name}</cyan>:<cyan>{function}</cyan>@<cyan>{line}</cyan>: " "<level>{message}</level>") else: log_level = "INFO" log_fmt = ("[decofre] " "<green>{time:YYYY-MM-DD}T{time:HH:mm:ss}</green> {level}: " "<level>{message}</level>") logger.add(utils.TqdmCompatibleStream(), level=log_level, format=log_fmt, colorize=True) # Since there are no support for default positional arguments in # docopt yet. Might be useful for complex default values, too if arguments["<out-file>"] is None: arguments["<out-file>"] = "-" corpus_dir = pathlib.Path(arguments["<corpus-path>"]).resolve() antecedents_dir = (corpus_dir / "antecedents").resolve() with directory_manager( arguments["--intermediate-dir"]) as intermediate_dir: score_dir = (intermediate_dir / "score").resolve() gold_clusters_dir = (corpus_dir / "clusters").resolve() sys_clusters_dir = (intermediate_dir / "clusters").resolve() for p in (score_dir, sys_clusters_dir): p.mkdir(parents=True, exist_ok=True) antecedents_files = sorted(antecedents_dir.glob("*.json")) pbar = tqdm.tqdm( antecedents_files, unit="documents", desc="Processing", unit_scale=True, unit_divisor=1024, dynamic_ncols=True, leave=False, disable=None, ) for data_file in pbar: stem = data_file.stem pbar.set_description(f"Processing {stem}") score_file = score_dir / f"{stem}.json" if not score_file.exists() or arguments["--overwrite"]: score.main_entry_point( [arguments["<model>"], str(data_file), str(score_file)]) else: logger.debug(f"Skipping scoring {score_file}") sys_clusters_file = sys_clusters_dir / f"{stem}.json" if not sys_clusters_file.exists() or arguments["--overwrite"]: clusterize.main_entry_point( [str(score_file), str(sys_clusters_file)]) else: logger.debug(f"Skipping clustering {sys_clusters_file}") with smart_open(arguments["<out-file>"], "w") as out_stream: out_stream.writelines( scorch.main.process_dirs(gold_clusters_dir, sys_clusters_dir))
""" @author: 闲欢 """ from loguru import logger logger.debug('this is a debug message') logger.add('hello.log') logger.debug('i am in log file') id = logger.add('world.log', format="{time} | {level} | {message}", level="INFO") logger.info('this is a debug message') logger.remove(id) logger.info('this is another debug message') logger.add('runtime.log') logger.info('this is an debug message') # 超过200M就新生成一个文件 logger.add("size.log", rotation="200 MB") # 每天中午12点生成一个新文件 logger.add("time.log", rotation="12:00") # 一周生成一个新文件 logger.add("size.log", rotation="1 week") @logger.catch def a_function(x): return 1 / x
def main(argv=None): import argparse parser = argparse.ArgumentParser( description="PINT tool for simulating TOAs", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument("parfile", help="par file to read model from") parser.add_argument("timfile", help="Output TOA file name") parser.add_argument( "--inputtim", help="Input tim file for fake TOA sampling", type=str, default=None, ) parser.add_argument( "--startMJD", help="MJD of first fake TOA", type=float, default=56000.0, ) parser.add_argument("--ntoa", help="Number of fake TOAs to generate", type=int, default=100) parser.add_argument("--duration", help="Span of TOAs to generate (days)", type=float, default=400.0) parser.add_argument("--obs", help="Observatory code", default="GBT") parser.add_argument( "--freq", help="Frequency for TOAs (MHz)", nargs="+", type=float, default=1400.0, ) parser.add_argument( "--error", help="Random error to apply to each TOA (us)", type=float, default=1.0, ) parser.add_argument( "--addnoise", action="store_true", default=False, help="Actually add in random noise, or just populate the column", ) parser.add_argument( "--fuzzdays", help="Standard deviation of 'fuzz' distribution (jd)", type=float, default=0.0, ) parser.add_argument("--plot", help="Plot residuals", action="store_true", default=False) parser.add_argument("--format", help="The format of out put .tim file.", default="TEMPO2") parser.add_argument( "--log-level", type=str, choices=("TRACE", "DEBUG", "INFO", "WARNING", "ERROR"), default=pint.logging.script_level, help="Logging level", dest="loglevel", ) args = parser.parse_args(argv) log.remove() log.add( sys.stderr, level=args.loglevel, colorize=True, format=pint.logging.format, filter=pint.logging.LogFilter(), ) log.info("Reading model from {0}".format(args.parfile)) m = pint.models.get_model(args.parfile) out_format = args.format error = args.error * u.microsecond if args.inputtim is None: log.info("Generating uniformly spaced TOAs") ts = pint.simulation.make_fake_toas_uniform( startMJD=args.startMJD, endMJD=args.startMJD + args.duration, ntoas=args.ntoa, model=m, obs=args.obs, error=error, freq=np.atleast_1d(args.freq) * u.MHz, fuzz=args.fuzzdays * u.d, add_noise=args.addnoise, ) else: log.info("Reading initial TOAs from {0}".format(args.inputtim)) ts = pint.simulation.make_fake_toas_fromtim( args.inputtim, model=m, obs=args.obs, error=error, freq=np.atleast_1d(args.freq) * u.MHz, fuzz=args.fuzzdays * u.d, add_noise=args.addnoise, ) # Write TOAs to a file ts.write_TOA_file(args.timfile, name="fake", format=out_format) if args.plot: # This should be a very boring plot with all residuals flat at 0.0! import matplotlib.pyplot as plt from astropy.visualization import quantity_support quantity_support() r = pint.residuals.Residuals(ts, m) plt.errorbar( ts.get_mjds(), r.calc_time_resids(calctype="taylor").to(u.us), yerr=ts.get_errors().to(u.us), fmt=".", ) plt.xlabel("MJD") plt.ylabel("Residual (us)") plt.grid(True) plt.show()
import pyutil.pretty as pretty import routers if __name__ == "__main__": # disable stdin, redirect stdout/stderr to uhrs.out stdout = open(file="uhrs.out", mode="a", encoding="utf-8") stdin = open(file="/dev/null") os.dup2(stdin.fileno(), sys.stdin.fileno()) os.dup2(stdout.fileno(), sys.stderr.fileno()) os.dup2(stdout.fileno(), sys.stdout.fileno()) # become a daemon process daemon.daemonize(pid_file="uhrs.pid") # init loguru logger.remove(None) # logger.add(sink="uhrs_{time}.log", rotation="00:00", retention="10 days") logger.add(sink="uhrs.log") logger.info("starting uhrs service...") conf = configparser.ConfigParser() conf.read(filenames="uhrs.cfg", encoding="utf-8") log_num_list = conf.get("server", "log_num_list", fallback="") for num in log_num_list.split(","): pretty.set_log_number(int(num)) # init db pool dbpool.init_db_pool(conf) # init redis redis_on = conf.has_section("redis") if redis_on:
"""A loguru Helper method to format log strings""" assert isinstance(record, dict) keyname = "extra[classname]" if "classname" in record.get("extra") else "name" return ( "<green>{time:YYYY-MM-DD_HH:mm:ss.SSS}</green> | <lvl>{level: <8}</lvl> | <c>%s:{function}:{line}</c> - <level>{message}</level>" % keyname ) def log_retention(files, max_log_size=20*1024**3): """Specify logfile retention policy, per file""" stats = [(_file, os.stat(_file)) for _file in files] stats.sort(key=lambda s: -s[1].st_mtime) # Sort files from newest to oldest while sum(s[1].st_size for s in stats) > max_log_size: _file, _ = stats.pop() os.remove(_file) logger.remove() # Disable intrusive loguru defaults... ref # https://github.com/Delgan/loguru/issues/208 logger.add( sink=sys.stderr, colorize=True, diagnose=True, backtrace=True, enqueue=True, serialize=False, catch=True, level="DEBUG", # compression, encoding, and retention throw errors in logger.add() #compression="gzip", #encoding="utf-8", #retention="3 months", # also see log_retention() # log_format_string(record)
def test_handler_removed(): i = logger.add(sys.__stdout__) logger.add(sys.__stderr__) logger.remove(i) assert repr( logger) == "<loguru.logger handlers=[(id=1, level=10, sink=<stderr>)]>"
r = Repository(os.path.join("artifacts", f"{REPO_NAME}.db.tar.zst")) with open(os.path.join('artifacts', 'repoPackages.json'), 'w') as writer: vs = list(map(lambda v: v.dict(), r.entries.values())) writer.write(json.dumps(vs)) repo.upload_file('repoPackages.json') def render_main(): repo = S3Repo(REPO_NAME, BUCKET_NAME) repo.download() upload_index(repo) if __name__ == "__main__": logger.remove(0) logger.add(sys.stderr, format="<level>{level: <8}</level> | <cyan>{function}</cyan>:" "<cyan>{line}</cyan> - <level>{message}</level>", colorize=True) import argparse parser = argparse.ArgumentParser() parser.add_argument('--build', help='build latest packages', action='store_true') parser.add_argument('--package', help='update repo with latest packages', action='store_true') parser.add_argument('--render', help='render and upload an index.html for the repo',
def function(): i = logger.add(NonSafeSink(0.1), format="{message}", catch=False) logger.debug("AAA") logger.info("BBB") logger.success("CCC") logger.remove(i)
def wrapper(*args, **kwargs): state = cur_ctx().find_object(self.LogState) # get the verbose/quiet levels from context if state.verbose: log_level = "DEBUG" elif state.quiet: log_level = "ERROR" else: log_level = self._stderr_log_level logger.remove() # remove existing default logger logger.add( sys.stderr, level=log_level, format=self.stderr_format_func ) if logfile and state.logfile: # start a log file # If a subcommand was used, log to a file in the # logs/ subdirectory with the subcommand in the file name. if log_dir_parent is not None: self._log_dir_parent = log_dir_parent if self._log_dir_parent is None: log_dir_path = Path(".") / "logs" else: log_dir_path = Path(self._log_dir_parent) subcommand = cur_ctx().invoked_subcommand if subcommand is None: subcommand = state.subcommand if subcommand is not None: logfile_prefix = f"{self._name}-{subcommand}" else: logfile_prefix = f"{self._name}" if log_dir_path.exists(): log_numbers = [ f.name[len(logfile_prefix) + 1 : -4] for f in log_dir_path.glob( logfile_prefix + "_*.log" ) ] log_number_ints = sorted( [int(n) for n in log_numbers if n.isnumeric()] ) if len(log_number_ints) > 0: log_number = log_number_ints[-1] + 1 if ( self._retention is not None and len(log_number_ints) > self._retention ): for remove in log_number_ints[ : len(log_number_ints) - self._retention ]: ( log_dir_path / f"{logfile_prefix}_{remove}.log" ).unlink() else: log_number = 0 else: log_number = 0 if self._retention == 0: state.logfile_path = ( log_dir_path / f"{logfile_prefix}.log" ) else: state.logfile_path = ( log_dir_path / f"{logfile_prefix}_{log_number}.log" ) state.logfile_handler_id = logger.add( str(state.logfile_path), level=self._file_log_level ) logger.debug(f'Command line: "{" ".join(sys.argv)}"') logger.debug(f"{self._name} version {self._version}") logger.debug( f"Run started at {str(self.start_times['Total']['wall'])[:SKIP_FIELDS]}" ) return user_func(*args, **kwargs)
def main(): start_time = time() file_counter = 0 ############################# # logging if not os.path.exists("./log"): os.mkdir("./log") logger.remove() # don't put messages into notebook output logger.add("./log/apobec_snp_rate_log_{time}.txt", backtrace=False) logger.add(sys.stderr, level="CRITICAL") ############################################ if os.path.exists("./input_data"): input_files = get_input_files_names("./input_data") num_files = len(input_files) print(""" --------------- job started at {0} ... --------------- """.format(get_current_time())) print("collecting snp...") progress_bar = IntProgress(min=0, max=num_files, bar_style='success') display(progress_bar) series_container = [] for f in input_files: try: ref_seq = get_ref(f) snpes_per_read = count_snp_per_read(ref_seq, f) series_container.append(snpes_per_read) progress_bar.value += 1 file_counter += 1 except Exception: logger.info( """\n\t================================================== file: {0} ====================================================""". format(f)) logger.exception("") print("exception detected. see log for more details") progress_bar.value += 1 print("snp collected...") try: df_raw_count = pd.DataFrame(series_container).T # drop first raw which is a reference to itself comparison df_raw_count.drop([0], inplace=True) pivot = df_raw_count.describe() mode, median = get_mode_median(df_raw_count) pivot_df = pd.concat([pivot, mode, median]) # renaming index values in pivot df pivot_df.rename(index={ 'count': 'reads total', 'mean': 'mean snp per read', 'min': 'min number of snp', 'max': 'max number of snp', 'mode': 'mode snp', 'median': 'median snp' }, inplace=True) pivot_df = calculate_non_mutated(pivot_df, df_raw_count).T time_stamp = get_current_time() create_strip_plot(df_raw_count, time_stamp) print("plot consctructed...") df_raw_count = df_raw_count.T # transpose after plotting save_df(pivot_df, df_raw_count, time_stamp) print("spreadsheet saved...") finish_time = time() total_time = finish_time - start_time show_report(total_time, file_counter) except Exception: logger.info( """\n\t==================================================""") logger.exception("") print("exception detected. see log for more details") else: os.mkdir("./input_data") print(""" Houston, we have a problem... -------- folder 'input_data' doesn't exist in the current directory or may be you've created it but misspelled its name. Anyway, it has just been created by this script. Paste your 'fasta' files into the folder 'input_data' and run this script again. -------- """)
import os os.environ.setdefault("LOGURU_INFO_COLOR", "<b>") os.environ.setdefault("LOGURU_DEBUG_COLOR", "<b>") os.environ.setdefault("LOGURU_ERROR_COLOR", "<b>") os.environ.setdefault("LOGURU_WARNING_COLOR", "<b>") from loguru import logger logger.remove(handler_id=None) logger.add("russia_{time}.log", rotation="100MB", encoding="utf-8", colorize=False)
Time, ensure_datetime, get_masked_array, get_unmasked, replace_cube_coord, shorten_columns, shorten_features, ) if "TQDMAUTO" in os.environ: from tqdm.auto import tqdm else: from tqdm import tqdm loguru_logger.enable("alepython") loguru_logger.remove() loguru_logger.add(sys.stderr, level="WARNING") logger = logging.getLogger(__name__) enable_logging("jupyter") warnings.filterwarnings("ignore", ".*Collapsing a non-contiguous coordinate.*") warnings.filterwarnings("ignore", ".*DEFAULT_SPHERICAL_EARTH_RADIUS.*") warnings.filterwarnings("ignore", ".*guessing contiguous bounds.*") warnings.filterwarnings( "ignore", 'Setting feature_perturbation = "tree_path_dependent".*' ) normal_coast_linewidth = 0.3 mpl.rc("figure", figsize=(14, 6))
def cli(log_level: str): logger.remove() logger.add(sys.stdout, level=log_level)
import os import argparse import sys import getpass import json import cgi import datetime from urllib.parse import unquote D2L_BASEURL = "https://mycourses.rit.edu/" # Not sure if this is unique to me, or just unique to RIT's tenant OU = 6605 logger.remove() logger.add(sys.stderr, level="INFO") # basically, mkdir -p /blah/blah/blah def mkdir_recursive(path): try: os.makedirs(path, exist_ok=True) except Exception as e: logger.error("Exception: {}".format(e)) exit(1) def get_xfrs_token(page_html): """ Method to parse a D2L page to find the XSRF.Token. The token is returned as a string :param page_html:
def init_logger(cls): logger.remove() logger.add(sys.stdout, colorize=True, format="{message}")