def main(ctx, verbose=False, debug=False, log_file=None, env_prefix=None, global_urls_variable=None): formatter = daiquiri.formatter.ColorFormatter( fmt="%(color)s%(levelname)s " "[%(name)s] %(message)s%(color_stop)s") outputs = [ daiquiri.output.Stream(sys.stderr, formatter=formatter) ] if log_file: outputs.append(daiquiri.output.File(log_file, formatter=formatter)) ctx.obj = { "debug": debug, } if env_prefix is not None: ctx.obj['env_prefix'] = env_prefix if global_urls_variable is not None: ctx.obj['global_urls_variable'] = global_urls_variable if debug: level = logging.DEBUG elif verbose: level = logging.INFO else: level = logging.WARNING daiquiri.setup(outputs=outputs, level=level)
def setup_logging(verbose=0): global logging progname = basename(sys.argv[0]) daiquiri.setup( level=(logging.ERROR - 10*verbose), program_name=progname) logging = daiquiri.getLogger(progname)
def create_logger(self, log_level=logging.INFO): daiquiri.setup( outputs=( daiquiri.output.File( directory=self.settings.get('LOG_PATH'), program_name=self.project_name, ), daiquiri.output.STDOUT, ) ) daiquiri.getLogger(program_name=self.project_name).logger.level = log_level self.logger = daiquiri.getLogger( program_name=self.project_name, log_level=log_level )
def init_log_info(log_path, log_filename): if not os.path.isdir(log_path): os.mkdir(log_path) log_file = log_path + "/" + log_filename daiquiri.setup( level=logging.INFO, outputs=( daiquiri.output.Stream(formatter=daiquiri.formatter.ColorFormatter( fmt="%(asctime)s [PID %(process)d] [%(levelname)s] " "%(name)s:%(lineno)d %(message)s")), daiquiri.output.File(log_file, formatter=daiquiri.formatter.ColorFormatter( fmt="%(asctime)s [PID %(process)d] " "[%(levelname)s] %(name)s:%(lineno)d " "%(message)s")), ) )
def configure_logging(self): formatter = daiquiri.formatter.ColorFormatter( fmt="%(color)s%(levelname)s " "[%(name)s] %(message)s%(color_stop)s") outputs = [ daiquiri.output.Stream(sys.stderr, formatter=formatter) ] if self.options.log_file: outputs.append(daiquiri.output.File(self.options.log_file, formatter=formatter)) if self.options.debug: level = logging.DEBUG elif self.options.verbose_level == 1: level = logging.INFO else: level = logging.WARNING daiquiri.setup(outputs=outputs, level=level)
import logging import daiquiri import daiquiri.formatter daiquiri.setup(level=logging.INFO, outputs=[ daiquiri.output.Stream( formatter=daiquiri.formatter.ColorExtrasFormatter( fmt=(daiquiri.formatter.DEFAULT_FORMAT + " [%(subsystem)s is %(mood)s]" + "%(extras)s"), keywords=['mood', 'subsystem'], )) ]) logger = daiquiri.getLogger(__name__, subsystem="example") logger.info("It works and log to stderr by default with color!", mood="happy", arbitrary_context="included")
https://www.python.org/dev/peps/pep-0503/ * A Platform Tag for Portable Linux Built Distributions https://www.python.org/dev/peps/pep-0513/ * The Wheel Binary Package Format 1.0 https://www.python.org/dev/peps/pep-0427/ """ import os import sys import re import logging import click import daiquiri daiquiri.setup() _LOGGER = logging.getLogger(__name__) # Adjusted based on: https://www.python.org/dev/peps/pep-0427/ _WHEEL_RE = re.compile( "(?P<distribution>.+)-(?P<version>.+)(-(?P<build_tag>.+))?-(?P<python_tag>.+)-(?P<abi_tag>.+)-(?P<platform_tag>.+).whl" ) def _check_python_artifacts(package_dir: str) -> bool: """Check Python artifacts present in the corresponding package directory.""" any_error = False for package_artifact in os.listdir(package_dir): artifact_path = os.path.join(package_dir, package_artifact)
def main(): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers() subparsers.required = True subparsers.dest = "command" subparser = subparsers.add_parser("simplify") subparser.add_argument("input", type=str, help="Input tree sequence") subparser.add_argument("output", type=str, help="Input tree sequence") subparser.set_defaults(func=run_simplify) subparser = subparsers.add_parser("sequential-augment") subparser.add_argument("input", type=str, help="Input tree sequence") subparser.add_argument("--num-threads", type=int, default=0) subparser.add_argument("--seed", type=int, default=1) subparser.set_defaults(func=run_sequential_augment) subparser = subparsers.add_parser("combine-ukbb-1kg") subparser.add_argument("chromosome", type=str, help="chromosome stem") subparser.add_argument("--num-individuals", type=int, help="number of individuals to use", default=None) subparser.set_defaults(func=run_combine_ukbb_1kg) subparser = subparsers.add_parser("benchmark-tskit") subparser.add_argument("input", type=str, help="Input tree sequence") subparser.add_argument( "--num-variants", type=int, default=None, help="Number of variants to benchmark genotypes decoding performance on" ) subparser.set_defaults(func=run_benchmark_tskit) subparser = subparsers.add_parser("benchmark-vcf") subparser.add_argument("input", type=str, help="Input VCF") subparser.add_argument( "--num-variants", type=int, default=None, help="Number of variants to benchmark genotypes decoding performance on" ) subparser.set_defaults(func=run_benchmark_vcf) subparser = subparsers.add_parser("compute-1kg-ukbb-gnn") subparser.add_argument("input", type=str, help="Input tree sequence") subparser.add_argument("output", type=str, help="Filename to write CSV to.") subparser.add_argument("--num-threads", type=int, default=16) subparser.set_defaults(func=run_compute_1kg_ukbb_gnn) subparser = subparsers.add_parser("compute-ukbb-gnn") subparser.add_argument("input", type=str, help="Input tree sequence") subparser.add_argument("output", type=str, help="Filename to write CSV to.") subparser.add_argument("--num-threads", type=int, default=16) subparser.set_defaults(func=run_compute_ukbb_gnn) subparser = subparsers.add_parser("compute-1kg-gnn") subparser.add_argument("input", type=str, help="Input tree sequence") subparser.add_argument("output", type=str, help="Filename to write CSV to.") subparser.add_argument("--num-threads", type=int, default=16) subparser.set_defaults(func=run_compute_1kg_gnn) subparser = subparsers.add_parser("compute-sgdp-gnn") subparser.add_argument("input", type=str, help="Input tree sequence") subparser.add_argument("output", type=str, help="Filename to write CSV to.") subparser.add_argument("--num-threads", type=int, default=16) subparser.set_defaults(func=run_compute_sgdp_gnn) subparser = subparsers.add_parser("snip-centromere") subparser.add_argument("input", type=str, help="Input tree sequence") subparser.add_argument("output", type=str, help="Output tree sequence") subparser.add_argument("chrom", type=str, help="Chromosome name") subparser.add_argument("centromeres", type=str, help="CSV file containing centromere coordinates.") subparser.set_defaults(func=run_snip_centromere) daiquiri.setup(level="INFO") args = parser.parse_args() args.func(args)
from collections import deque import random import gym import numpy as np import tensorflow as tf import daiquiri daiquiri.setup(level=daiquiri.logging.DEBUG) logger = daiquiri.getLogger(__name__) RENDER = False class Neuro: def __init__(self): n_in = 4 n_hidden = 4 n_out = 2 initializer = tf.contrib.layers.variance_scaling_initializer() self.input_ = tf.placeholder(dtype=tf.float32, shape=[None, n_in]) hidden = tf.layers.dense(self.input_, n_hidden, activation=tf.nn.tanh) self.probs = tf.nn.softmax(tf.layers.dense(hidden, 2)) self.gradients = tf.gradients(self.probs[0, 0], tf.trainable_variables()) def run(self, input_arr): return self.sess.run(self.probs, feed_dict={self.input_: input_arr})
:Created: 8/23/20 """ import logging import hashlib import os from pathlib import Path import click import daiquiri cwd = os.path.dirname(os.path.realpath(__file__)) logfile = cwd + "/offline_checksum.log" daiquiri.setup(level=logging.DEBUG, outputs=( daiquiri.output.File(logfile), "stdout", )) logger = daiquiri.getLogger(__name__) def get_files(data: Path, ext: str = ""): f = list() if len(ext) > 0: ext = "." + ext.lstrip(".") files = data.rglob(f"*{ext}") for file in files: if Path(file).is_file(): f.append(file) return f
def main(debug=False): colorama.init() daiquiri.setup(level=logging.DEBUG if debug else logging.WARNING)
def prepare_service(args=None, conf=None, default_config_files=None, log_to_std=False, logging_level=None): if conf is None: conf = cfg.ConfigOpts() opts.set_defaults() # FIXME(jd) Use the pkg_entry info to register the options of these libs db_options.set_defaults(conf) policy_opts.set_defaults(conf) # Register our own Gnocchi options for group, options in opts.list_opts(): conf.register_opts(list(options), group=None if group == "DEFAULT" else group) conf.register_cli_opts(opts._cli_options) conf.set_default("workers", utils.get_default_workers(), group="metricd") conf(args, project='gnocchi', validate_default_values=True, default_config_files=default_config_files, version=pbr.version.VersionInfo('gnocchi').version_string()) if not log_to_std and (conf.log_dir or conf.log_file): outputs = [daiquiri.output.File(filename=conf.log_file, directory=conf.log_dir)] else: outputs = [daiquiri.output.STDERR] if conf.use_syslog: outputs.append( daiquiri.output.Syslog(facility=conf.syslog_log_facility)) if conf.use_journal: outputs.append(daiquiri.output.Journal()) daiquiri.setup(outputs=outputs) if logging_level is None: if conf.debug: logging_level = logging.DEBUG elif conf.verbose: logging_level = logging.INFO else: logging_level = logging.WARNING logging.getLogger("gnocchi").setLevel(logging_level) # HACK(jd) I'm not happy about that, fix AP class to handle a conf object? archive_policy.ArchivePolicy.DEFAULT_AGGREGATION_METHODS = ( conf.archive_policy.default_aggregation_methods ) # If no coordination URL is provided, default to using the indexer as # coordinator if conf.storage.coordination_url is None: if conf.storage.driver == "redis": conf.set_default("coordination_url", conf.storage.redis_url, "storage") elif conf.incoming.driver == "redis": conf.set_default("coordination_url", conf.incoming.redis_url, "storage") else: parsed = urlparse.urlparse(conf.indexer.url) proto, _, _ = parsed.scheme.partition("+") parsed = list(parsed) # Set proto without the + part parsed[0] = proto conf.set_default("coordination_url", urlparse.urlunparse(parsed), "storage") cfg_path = conf.oslo_policy.policy_file if not os.path.isabs(cfg_path): cfg_path = conf.find_file(cfg_path) if cfg_path is None or not os.path.exists(cfg_path): cfg_path = os.path.abspath(os.path.join(os.path.dirname(__file__), 'rest', 'policy.json')) conf.set_default('policy_file', cfg_path, group='oslo_policy') conf.log_opt_values(LOG, logging.DEBUG) return conf
import logging import pathlib import daiquiri from smart_simulation.cfg_templates.config import package_dir daiquiri.setup( level=logging.INFO, outputs=( daiquiri.output.Stream( formatter=daiquiri.formatter.ColorFormatter( fmt="%(asctime)s [%(levelname)s] %(name)s.%(" "funcName)s: %(message)s" ) ), ), ) PACKAGE_PATH = pathlib.Path(package_dir) SIMULATION_OUTPUTS_PATH = PACKAGE_PATH / "smart_simulation" / "outputs" / "simulations" SIMULATION_WEIGHTS_PATH = SIMULATION_OUTPUTS_PATH / "weights" SIMULATION_SERVINGS_PATH = SIMULATION_OUTPUTS_PATH / "servings" def weight_files(weights_directory: pathlib.PurePath = SIMULATION_WEIGHTS_PATH) -> list: """ Return a list of weight files paths from a directory. The local simulation output directory is the default. Args: weights_directory: Directory of the weight files. Returns: A list of paths of all files in the directory.
def __init__(self, database=None): daiquiri.setup(level=logging.INFO) self.logger = daiquiri.getLogger(__name__) self.database = Database() if not database else database self.faker = Faker() self.postal_codes = None
import os import shutil import random import numpy as np from tqdm import tqdm from collections import OrderedDict from config import DIR, TARGET_DIR, OUTPUT_DIR, NUM_TOP_i_IMAGES, NUM_TOP_p_Predictions import daiquiri as dqr import logging dqr.setup(level=logging.DEBUG) logger = dqr.getLogger() def maybe_sample_images(dataset_name: str, sample_num: int = 5): """Sample some images if the given directory is empty Args: dataset_name: dataset_name sample_num: number of images that randomly sampled if there is no images or the directory does not exist at all. """ target_dir = os.path.join(TARGET_DIR, dataset_name) # Create directory if not exist if not os.path.exists(target_dir): logger.warning( 'Target directory does not exist. Creating directory {}'.format( target_dir)) os.makedirs(target_dir)
from flask import request, Blueprint, jsonify, current_app from IGitt.GitHub.GitHubRepository import GitHubRepository from IGitt.GitHub.GitHubIssue import GitHubToken, GitHubIssue from sesheta.utils import ( notify_channel, random_positive_emoji, random_positive_emoji2, calculate_pullrequest_size, set_size, ) from sesheta.webhook_processors.github_reviews import * from sesheta.webhook_processors.github_pull_requests import * from sesheta.webhook_processors.github_issue_analyzer import analyse_github_issue daiquiri.setup(level=logging.DEBUG, outputs=("stdout", "stderr")) _LOGGER = daiquiri.getLogger(__name__) _RELATED_REGEXP = r"\w+:\ \#([0-9]+)" _DRY_RUN = os.environ.get("SESHETA_DRY_RUN", False) _SESHETA_GITHUB_ACCESS_TOKEN = os.getenv("SESHETA_GITHUB_ACCESS_TOKEN", None) _SESHETA_GITHUB_WEBHOOK_SECRET = os.getenv("SESHETA_GITHUB_WEBHOOK_SECRET", None) _GIT_API_REQUEST_HEADERS = { "Authorization": "token %s" % _SESHETA_GITHUB_ACCESS_TOKEN } webhooks = Blueprint("webhook", __name__, url_prefix="") def handle_github_open_issue(issue: dict, repository: dict) -> None: # pragma: no cover
import contextlib from pathlib import Path import daiquiri from _pygitviz import graphviz from _pygitviz import util from _pygitviz import git from _pygitviz.graphviz import git_to_dot daiquiri.setup( level=logging.WARNING, outputs=( daiquiri.output.Stream( sys.stdout, formatter=daiquiri.formatter.ColorFormatter( fmt="%(color)s[%(levelname)s] %(message)s%(color_stop)s" ), ), ), ) LOGGER = daiquiri.getLogger(__file__) def main() -> None: """Run the PyGitViz program.""" operating_system = util.get_os() parser = _create_parser(operating_system) args = parser.parse_args(sys.argv[1:]) with _convert_error_to_log(traceback=args.traceback):
import logging import daiquiri import daiquiri.formatter daiquiri.setup( level=logging.INFO, outputs=( daiquiri.output.Stream( formatter=daiquiri.formatter.ColorFormatter( fmt=(daiquiri.formatter.DEFAULT_FORMAT + " [%(subsystem)s is %(mood)s]") ) ), ), ) logger = daiquiri.getLogger(__name__, subsystem="example") logger.info("It works and log to stderr by default with color!", mood="happy")
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ import logging import daiquiri from pythonjsonlogger import jsonlogger stream_fmt = "%(asctime)s [%(process)d] %(color)s%(levelname)-8.8s %(name)s: %(message)s%(color_stop)s" json_fmt = "(asctime) (process) (levelname) (name) (message)" daiquiri.setup( level=logging.INFO, outputs=( daiquiri.output.Stream( formatter=daiquiri.formatter.ColorFormatter(stream_fmt), ), daiquiri.output.TimedRotatingFile( filename="{{ cookiecutter.package_name }}.log", formatter=jsonlogger.JsonFormatter(json_fmt, json_default=str), ), ), ) def client_logger(name=__name__): return daiquiri.getLogger(name)
from datetime import timedelta, datetime from pathlib import Path from shutil import rmtree import daiquiri import graphyte import common.config as config import common.helper as helper import common.monitor as monitor from common.monitor import send_series_event, s_events from common.constants import mercure_defs, mercure_folders daiquiri.setup( level=logging.INFO, outputs=(daiquiri.output.Stream( formatter=daiquiri.formatter.ColorFormatter( fmt="%(color)s%(levelname)-8.8s " "%(name)s: %(message)s%(color_stop)s")), ), ) logger = daiquiri.getLogger("cleaner") def terminate_process(signalNumber, frame): """Triggers the shutdown of the service.""" helper.g_log("events.shutdown", 1) logger.info("Shutdown requested") monitor.send_event(monitor.h_events.SHUTDOWN_REQUEST, monitor.severity.INFO) # Note: main_loop can be read here because it has been declared as global variable if "main_loop" in globals() and main_loop.is_running: main_loop.stop()
__version__ = 0.1 import os import logging import configparser import daiquiri import daiquiri.formatter _ROOT = os.path.dirname(os.path.abspath(__file__)) _CONFIG = os.path.join(_ROOT, 'config.ini') FORMAT = ("%(asctime)s :: %(color)s%(levelname)s :: %(name)s :: %(funcName)s :" "%(message)s%(color_stop)s") daiquiri.setup(level=logging.INFO, outputs=(daiquiri.output.Stream( formatter=daiquiri.formatter.ColorFormatter(fmt=FORMAT)), )) logger = daiquiri.getLogger("root") if not os.path.isfile(_CONFIG): logger.error("Configuration file '%s' not found", _CONFIG) config = None else: config = configparser.ConfigParser(allow_no_value=True) with open(_CONFIG) as fobj: config.read_file(fobj)
initialization=U_init) return final_centroids, indicator_vector_final if __name__ == "__main__": logger.info("Command line: " + " ".join(sys.argv)) log_memory_usage("Memory at startup") arguments = docopt.docopt(__doc__) paraman = ParameterManagerEfficientNystrom(arguments) initialized_results = dict((v, None) for v in lst_results_header) resprinter = ResultPrinter(output_file=paraman["--output-file_resprinter"]) resprinter.add(initialized_results) resprinter.add(paraman) has_failed = False if paraman["-v"] >= 2: daiquiri.setup(level=logging.DEBUG) elif paraman["-v"] >= 1: daiquiri.setup(level=logging.INFO) else: daiquiri.setup(level=logging.WARNING) logging.warning("Verbosity set to warning") logging.info("Verbosity set to info") logging.debug("Verbosity set to debug") try: dataset = paraman.get_dataset() if dataset["x_train"].dtype != np.float32: dataset["x_train"] = dataset["x_train"].astype(np.float32)
import os from flask import Flask, request from flask.json import jsonify from flask_cors import CORS import src.config as config from src.exceptions import HTTPError from fabric8a_auth.auth import AuthError from src.trained_model_details import trained_model_details from rudra.utils.validation import check_field_exists from rudra.deployments.emr_scripts.pypi_emr import PyPiEMR from rudra.deployments.emr_scripts.maven_emr import MavenEMR from rudra.deployments.emr_scripts.npm_emr import NpmEMR daiquiri.setup(level=os.environ.get('FLASK_LOGGING_LEVEL', logging.INFO)) _logger = daiquiri.getLogger(__name__) app = Flask(__name__) CORS(app) emr_instances = {'maven': MavenEMR, 'pypi': PyPiEMR, 'npm': NpmEMR} @app.route('/api/v1/readiness') def readiness(): """Readiness probe.""" return jsonify({"status": "ready"}), 200 @app.route('/api/v1/liveness')
from cyborg_regidores import __version__ as cyborg_regidores_version from cyborg_regidores.topic_names import ( GITHUB_WEBHOOK_TOPIC_NAME, GITLAB_WEBHOOK_TOPIC_NAME, JIRA_WEBHOOK_TOPIC_NAME, TRELLO_WEBHOOK_TOPIC_NAME, GOOGLE_CHATBOT_TOPIC_NAME, NORMALIZED_EVENTS_TOPIC_NAME, AICOE_ACTIVITY_TOPIC_NAME, ) DEBUG = os.getenv("DEBUG", True) daiquiri.setup() _LOGGER = daiquiri.getLogger("webhook2kafka") _LOGGER.setLevel(logging.DEBUG if DEBUG else logging.INFO) _KAFAK_BOOTSTRAP_SERVERS = os.getenv("KAFKA_BOOTSTRAP_SERVERS", "localhost:9092") if __name__ == "__main__": _LOGGER.info(f"Cyborg Regidores dump v{cyborg_regidores_version}.") _LOGGER.debug("DEBUG mode is enabled!") _LOGGER.debug(f"using Kafka bootstrap servers: {_KAFAK_BOOTSTRAP_SERVERS}") # let's get all that we got... consumer = KafkaConsumer( AICOE_ACTIVITY_TOPIC_NAME, bootstrap_servers=_KAFAK_BOOTSTRAP_SERVERS,
# SrcMtrcs # Copyright(C) 2018 Christoph Görn # # This program is free software: you can redistribute it and / or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. """This is Source Operations Metrics...""" import logging import daiquiri from srcmtrcs import create_application, __version__ daiquiri.setup(level=logging.DEBUG, outputs=('stdout', 'stderr')) _LOGGER = daiquiri.getLogger(__name__) _LOGGER.info(f"Hi, I am the Source Operations Metrics bot, " f"and I'm running v{__version__}") application = create_application()
def main(input_file, regions_file, output_directory, input_signature, elements_file, elements, genome, element_mutations, cluster_mutations, smooth_window, cluster_window, kmer, n_simulations, simulation_mode, simulation_window, signature_calculation, signature_group, cores, seed, log_level, concatenate, clustplot, qqplot, gzip ): """ OncodriveCLUSTL is a sequence based clustering method to identify cancer drivers across the genome Args: input_file (str): path to mutations file regions_file (str): path to input genomic coordinates file output_directory(str): path to output directory. Output files will be generated in it. input_signature (str): path to file containing input context based mutational probabilities. By default (when no input signatures), OncodriveCLUSTL will calculate them from the mutations input file. elements_file (str): path to file containing one element per row (optional) to analyzed the listed elements. By default, OncodriveCLUSTL analyzes all genomic elements contained in `regions_file`. elements (str): genomic element symbol (optional). The analysis will be performed only on the specified GEs. genome (str): genome to use: 'hg38', 'hg19', 'mm10', 'c3h', 'car', 'cast' and 'f344' element_mutations (int): minimum number of mutations per genomic element to undertake analysis cluster_mutations (int): minimum number of mutations to define a cluster smooth_window (int): Tukey kernel smoothing window length cluster_window (int): clustering window length kmer (int): context nucleotides to calculate the mutational probabilities (trinucleotides or pentanucleotides) n_simulations (int): number of simulations simulation_mode (str): simulation mode simulation_window (int): window length to simulate mutations signature_calculation (str): signature calculation, mutation frequencies (default) or mutation counts normalized by k-mer region counts signature_group (str): header of the column to group signatures. One signature will be computed for each group cores (int): number of CPUs to use seed (int): seed log_level (str): verbosity of the logger concatenate (bool): flag to calculate clustering on collapsed genomic regions (e.g., coding regions in a gene) clustplot (bool): flag to generate a needle plot with clusters for an element qqplot (bool): flat to generate a quantile-quantile (QQ) plot for a dataset gzip (bool): flag to generate GZIP compressed output files Returns: None """ global logger # Get output directory if not os.path.exists(output_directory): os.makedirs(output_directory, exist_ok=True) # Get cache directory path_cache = os.path.join(output_directory, 'cache') os.makedirs(path_cache, exist_ok=True) # Get output files name if elements_file is not None: output_file = elements_file.split('/')[-1] else: output_file = 'results' if gzip: elements_output_file = 'elements_{}.txt.gz'.format(output_file) clusters_output_file = 'clusters_{}.tsv.gz'.format(output_file) else: elements_output_file = 'elements_{}.txt'.format(output_file) clusters_output_file = 'clusters_{}.tsv'.format(output_file) daiquiri.setup(level=LOGS[log_level], outputs=( daiquiri.output.STDERR, daiquiri.output.File( filename=output_file + '.log', directory=output_directory ) )) logger = daiquiri.getLogger() # suppress log messages from some libraries daiquiri.getLogger('bgdata').setLevel(logging.WARNING) daiquiri.getLogger('bgsignature').setLevel(logging.WARNING) logger.info('OncodriveCLUSTL') logger.info('\n'.join([ '', 'input_file: {}'.format(input_file), 'regions_file: {}'.format(regions_file), 'input_signature: {}'.format(input_signature), 'output_directory: {}'.format(output_directory), 'genome: {}'.format(genome), 'element_mutations: {}'.format(element_mutations), 'cluster_mutations: {}'.format(cluster_mutations), 'concatenate: {}'.format(concatenate), 'smooth_window: {}'.format(smooth_window), 'cluster_window: {}'.format(cluster_window), 'k-mer: {}'.format(kmer), 'simulation_mode: {}'.format(simulation_mode), 'simulation_window: {}'.format(simulation_window), 'n_simulations: {}'.format(n_simulations), 'signature_calculation: {}'.format(signature_calculation), 'signature_group: {}'.format(signature_group), 'cores: {}'.format(cores), 'gzip: {}'.format(gzip), 'seed: {}'.format(seed) ])) logger.info('Initializing OncodriveCLUSTL...') # Check parameters if simulation_window == 31 and smooth_window == 11 and cluster_window == 11: logger.warning( '\nRunning with default simulating, smoothing and clustering OncodriveCLUSTL parameters. ' 'Default parameters may not be optimal for your data.\n' 'Please, read Supplementary Methods to perform model selection for your data.' ) if not input_signature and signature_calculation == 'frequencies': logger.warning( '\nSignatures will be calculated as mutation frequencies: ' '# mutated ref>alt k-mer counts / # total substitutions\n' 'Please, read Supplementary Methods to perform a more accurate signatures calculation' ) if signature_calculation == 'region_normalized': logger.warning( '\nMutation k-mer counts will be normalized by k-mer region counts in {}\n' 'Only mutations inside regions will contribute for the signature calculation'.format(regions_file) ) if n_simulations < 1000: raise excep.UserInputError('Invalid number of simulations: please choose an integer greater than 1000') if clustplot: if len(elements) > 10: raise excep.UserInputError('Needle plots can only be generated for a maximum of 10 elements') # Create a list of elements to analyze if elements is not None: elements = set(elements) if elements_file is not None: elements |= set([line.strip().split()[0] for line in open(elements_file, 'r')]) if elements is None and elements_file is None: elements = set([]) if elements: logger.info( 'Input element{}: {}'.format('s' if len(elements) > 1 else '', len(elements)) ) logger.info(', '.join(elements)) # Parse regions and dataset mutations logger.info('Parsing genomic regions and mutations...') regions_d, concat_regions_d, chromosomes_d, strands_d, mutations_d, samples_d, groups_d = pars.parse( regions_file, elements, input_file, concatenate, signature_group ) # Summary mut = 0 elem = 0 element_mutations_cutoff = False for k, v in mutations_d.items(): mut += len(v) elem += 1 if not element_mutations_cutoff: if len(v) >= element_mutations: element_mutations_cutoff = True logger.info('Validated elements in genomic regions: {}'.format(len(regions_d.keys()))) logger.info('Validated elements with mutations: {}'.format(elem)) logger.info('Total substitution mutations: {}'.format(mut)) if not element_mutations_cutoff: raise excep.UserInputError('No element found with enough mutations to perform analysis') # Signature file_prefix = input_file.split('/')[-1].split('.')[0] path_pickle = os.path.join(path_cache, '{}_kmer_{}.pickle'.format(file_prefix, kmer)) if not input_signature: """ Calculate signatures By default, all substutions are taken into account to calculate the relative frequencies for each k-mer ref>alt. Alternatively, when specified through '--signature-calculation region_normalized', k-mer mutation counts can be normalized by the k-mer counts in the regions under analysis listed in 'regions_file'. In this case, only substitutions that fall inside the regions will contribute to the signature calculation. For both options, k-mers are not collapsed (192 channels) and do not include N (unknown reference nucleotides). """ logger.info('Computing signature{}...'.format('s for each group' if signature_group else '')) if signature_calculation == 'region_normalized': normalize_regions_file, signature_calc_function = regions_file, bgsign.normalize else: normalize_regions_file, signature_calc_function = None, bgsign.relative_frequency signatures_dict = signature_calc_function( mutations_file=input_file, regions_file=normalize_regions_file, kmer_size=int(kmer), genome_build=genome, collapse=None, group=signature_group, cores=cores ) # Reformat dictionary if not signature_group: signatures_to_pickle = {file_prefix: signatures_dict} else: signatures_to_pickle = signatures_dict # Save to cache with open(path_pickle, 'wb') as fd: pickle.dump(signatures_to_pickle, fd, protocol=2) logger.info('Signature{} computed'.format('s' if signature_group else '')) else: try: load_sign = bgsign.file.load(file=input_signature) except UnicodeDecodeError: raise excep.UserInputError('Error in input signatures file {}\n' 'Please, check signatures file format (JSON)'.format(input_signature)) # Check format and save pickle to cache keys = set(load_sign.keys()) if not signature_group: # Expects 'file_prefix' to be a key in signatures dictionary of dictionaries if file_prefix in keys: with open(path_pickle, 'wb') as fd: pickle.dump(load_sign, fd, protocol=2) # When dictionary has k-mer (ex. AAA>T) as keys, accepts dictionary and adds extra key for 'file_prefix' elif '>' in list(keys)[0]: with open(path_pickle, 'wb') as fd: pickle.dump({file_prefix: load_sign}, fd, protocol=2) # Error, 'file_prefix' and k-mers not found as keys, check format else: raise excep.UserInputError('Incorrect format for input signature file {}'.format(input_signature)) else: if not '>' in list(keys)[0]: with open(path_pickle, 'wb') as fd: pickle.dump(load_sign, fd, protocol=2) # n signature dictionaries are expected (n = groups) else: raise excep.UserInputError('Groups are missing in signature dictionary at {}'.format(input_signature)) logger.info('Input signature{} ready'.format('s' if signature_group else '')) # Initialize Experiment class variables and run elements_results, clusters_results, global_info_results = exp.Experiment( regions_d, concat_regions_d, chromosomes_d, strands_d, mutations_d, samples_d, genome, groups_d, path_pickle, element_mutations, cluster_mutations, smooth_window, cluster_window, int(kmer), n_simulations, simulation_mode, simulation_window, cores, clustplot, seed ).run() # Write elements results (returns list of ranked elements) sorted_list_elements = postp.write_element_results( genome=genome, results=(elements_results, global_info_results), directory=output_directory, file=elements_output_file, is_gzip=gzip ) logger.info('Elements results calculated') # Write clusters results postp.write_cluster_results( genome=genome, results=(clusters_results, global_info_results), directory=output_directory, file=clusters_output_file, sorter=sorted_list_elements, is_gzip=gzip ) logger.info('Clusters results calculated') # Cluster plot if clustplot: info_cluster_plots = cplot.make_clustplot( elements_results, clusters_results, global_info_results, directory=output_directory ) for message in info_cluster_plots: logger.info(message) # Quantile-quantile plot if qqplot: if len(elements) < 30: logger.warning('QQ-plot generated for less than 30 elements') input_qqplot_file = os.path.join(output_directory, elements_output_file) output_qqplot_file = os.path.join(output_directory, 'quantile_quantile_plot.png') qplot.make_qqplot( file=input_qqplot_file, output=output_qqplot_file, col_values='P_ANALYTICAL', top=10 ) logger.info('QQ-plot plot generated at : {}'.format(output_qqplot_file)) logger.info('Finished')
"""Get all titles currently in the RSS feeds for testing purposes.""" import configparser import logging import daiquiri from cds_paper_bot import read_feed, format_title daiquiri.setup(level=logging.ERROR) def load_config(feed_file): """Load configs into dict.""" config_dict = {} config = configparser.RawConfigParser() # load the feed config config.read(feed_file) config_dict = {} for key in config: config_dict[key] = config[key] return config_dict def main(): """Load the feeds, print all titles in a format useful for dumping into test_format_title.py.""" config = load_config("feeds.ini") # print(config) for experiment in config: for pub_type in config[experiment]: print(f" # {experiment} {pub_type}") this_feed = read_feed(config[experiment][pub_type]) if this_feed: this_feed_entries = this_feed["entries"]
You should have received a copy of the GNU General Public License along with this program. If not, see <http://www.gnu.org/licenses/>. """ import logging import sys import daiquiri import numpy as np from recommendation_engine.config.params_scoring import ScoringParams from recommendation_engine.config.path_constants import PMF_MODEL_PATH, PACKAGE_TAG_MAP, \ TRAINING_DATA_ITEMS, PRECOMPUTED_STACKS, ID_TO_PACKAGE_MAP, PACKAGE_TO_ID_MAP from recommendation_engine.model.pmf_prediction import PMFScoring from recommendation_engine.predictor.abstract_recommender import AbstractRecommender from recommendation_engine.utils.fileutils import load_rating daiquiri.setup(level=logging.WARNING) _logger = daiquiri.getLogger(__name__) class PMFRecommendation(AbstractRecommender): """Online recommendation logic. This class contains the online recommendation logic that will be used to score packages to the user's preferences at runtime. We need to run a single step of PMF and multiply the obtained user vector with the precomputed latent item vectors. """ def __init__(self, M, data_store, num_latent=ScoringParams.num_latent_factors):
def prepare_service(args=None, conf=None, default_config_files=None, log_to_std=False, logging_level=None, skip_log_opts=False): if conf is None: conf = cfg.ConfigOpts() # FIXME(jd) Use the pkg_entry info to register the options of these libs db_options.set_defaults(conf) # Register our own Gnocchi options for group, options in opts.list_opts(): conf.register_opts(list(options), group=None if group == "DEFAULT" else group) conf.register_cli_opts(opts._cli_options) workers = utils.get_default_workers() conf.set_default("workers", workers, group="metricd") conf.set_default("parallel_operations", workers) conf(args, project='gnocchi', validate_default_values=True, default_config_files=default_config_files, version=gnocchi.__version__) utils.parallel_map.MAX_WORKERS = conf.parallel_operations if not log_to_std and (conf.log_dir or conf.log_file): outputs = [daiquiri.output.File(filename=conf.log_file, directory=conf.log_dir)] else: outputs = [daiquiri.output.STDERR] if conf.use_syslog: outputs.append( daiquiri.output.Syslog(facility=conf.syslog_log_facility)) if conf.use_journal: outputs.append(daiquiri.output.Journal()) daiquiri.setup(outputs=outputs) if logging_level is None: if conf.debug: logging_level = logging.DEBUG elif conf.verbose: logging_level = logging.INFO else: logging_level = logging.WARNING logging.getLogger("gnocchi").setLevel(logging_level) # HACK(jd) I'm not happy about that, fix AP class to handle a conf object? archive_policy.ArchivePolicy.DEFAULT_AGGREGATION_METHODS = ( conf.archive_policy.default_aggregation_methods ) # If no coordination URL is provided, default to using the indexer as # coordinator if conf.coordination_url is None: if conf.storage.driver == "redis": conf.set_default("coordination_url", conf.storage.redis_url) elif conf.incoming.driver == "redis": conf.set_default("coordination_url", conf.incoming.redis_url) else: parsed = urlparse.urlparse(conf.indexer.url) proto, _, _ = parsed.scheme.partition("+") parsed = list(parsed) # Set proto without the + part parsed[0] = proto conf.set_default("coordination_url", urlparse.urlunparse(parsed)) if not skip_log_opts: LOG.info("Gnocchi version %s", gnocchi.__version__) conf.log_opt_values(LOG, logging.DEBUG) return conf
from soh.server.server import ApacheServer from soh.server.server import ApacheTomcatServer from soh.server.server import AuditServer from soh.server.server import AuthServer from soh.server.server import GmnServer from soh.server.server import JettyServer from soh.server.server import LdapServer from soh.server.server import PackageServer from soh.server.server import PortalServer from soh.server.server import Server from soh.server.server import SolrServer from soh.server.server import TomcatServer cwd = os.path.dirname(os.path.realpath(__file__)) logfile = cwd + "/health_check.log" daiquiri.setup(level=logging.WARN, outputs=(daiquiri.output.File(logfile), "stdout")) logger = daiquiri.getLogger("health_check.py: " + __name__) new_status = dict() async def check_read_only(hosts): for host in hosts: await do_read_only(host) async def do_read_only(host): st = datetime.now() host_ro = await server.read_only(host=host) logger.warning(f"Run time read only {host}: {datetime.now() - st}")
+-------------------+ [ r2 ] [ r3 ] .... """ import json import os import boto3 import daiquiri from wellcome_aws_utils import s3_utils, sns_utils from wellcome_aws_utils.lambda_utils import log_on_error daiquiri.setup(level=os.environ.get('LOG_LEVEL', 'INFO')) @log_on_error def main(event, _ctxt=None, s3_client=None, sns_client=None): topic_arn = os.environ["TOPIC_ARN"] s3_client = s3_client or boto3.client('s3') sns_client = sns_client or boto3.client('sns') s3_events = s3_utils.parse_s3_record(event=event) assert len(s3_events) == 1 s3_event = s3_events[0] resp = s3_client.get_object(Bucket=s3_event['bucket_name'], Key=s3_event['object_key'])
# import packages for data manipulation import pandas as pd import numpy as np import matplotlib import matplotlib.pyplot as plt import seaborn as sns import pickle import janitor from sklearn.preprocessing import StandardScaler from sklearn.model_selection import GroupKFold from sklearn.metrics import mean_squared_error, mean_squared_log_error, mean_absolute_error, r2_score from xgboost import XGBRegressor #logging package import daiquiri,logging daiquiri.setup(level=logging.INFO) logger = daiquiri.getLogger() #Loading data logger.info("Reading the data file...") df = pd.read_csv('../data/avocado.csv') logger.info("Quick preprocessing of data...") # Removing index column df.drop('Unnamed: 0', axis=1, inplace=True) # Removing records with TotalUS region, assuming it is nust the average of all other regions df = df.loc[df.region!='TotalUS'].reset_index(drop=True) # Making date to datetime and sorting chrinologically df['Date'] = pd.to_datetime(df['Date']) df = df.sort_values(['region','Date'])
"""Deeposlandia package """ from configparser import ConfigParser import logging import os import sys import daiquiri __version__ = '0.4' daiquiri.setup( level=logging.INFO, outputs=(daiquiri.output.Stream( formatter=daiquiri.formatter.ColorFormatter( fmt=("%(asctime)s :: %(levelname)s :: %(module)s :: " "%(funcName)s : %(color)s%(message)s%(color_stop)s"))), )) logger = daiquiri.getLogger("root") _DEEPOSL_CONFIG = os.getenv('DEEPOSL_CONFIG') _DEEPOSL_CONFIG = _DEEPOSL_CONFIG if _DEEPOSL_CONFIG is not None else "config.ini" config = ConfigParser() if os.path.isfile(_DEEPOSL_CONFIG): config.read(_DEEPOSL_CONFIG) else: logger.error("No file config.ini!") sys.exit(1)
"""Statistical methods used for analyzing the shared bike data """ import logging import daiquiri import numpy as np import pandas as pd from dateutil import parser from workalendar.europe import France from sklearn.cluster import KMeans import xgboost as xgb from jitenshea import config daiquiri.setup(logging.INFO) logger = daiquiri.getLogger("stats") # French Calendar cal = France() SEED = 2018 np.random.seed(SEED) CLUSTER_ACT_PATH_CSV ='jitenshea/data/cluster_activite.csv' ################################### ### CLUSTER ACTIVITE ###################################
import daiquiri import daiquiri.formatter import logging import sys import test_print_log_sub_module log_file = sys.path[0] + "/test.log" daiquiri.setup( level=logging.INFO, outputs=( daiquiri.output.Stream(formatter=daiquiri.formatter.ColorFormatter( fmt="%(asctime)s [PID %(process)d] [%(levelname)s] " "%(name)s:%(lineno)d %(message)s")), daiquiri.output.File(log_file, formatter=daiquiri.formatter.ColorFormatter( fmt="%(asctime)s [PID %(process)d] " "[%(levelname)s] %(name)s:%(lineno)d " "%(message)s")), ) ) logger = daiquiri.getLogger(__name__) if __name__ == "__main__": logger.info("log some info") logger.warning("log some warning") logger.error("log some error") logger.debug("log some debug") logger.info("some data is:%s" % ("testdata")) test_print_log_sub_module.print_log()
def setup_logging(): """Configure logging.""" daiquiri.setup(level=logging.DEBUG, outputs=(daiquiri.output.STDOUT, ))
# -*- coding: utf-8 -*- """:Mod: __init__ :Synopsis: :Author: servilla :Created: 12/30/17 """ import logging import os import sys sys.path.insert(0, os.path.abspath('..')) import daiquiri cwd = os.path.dirname(os.path.realpath(__file__)) logfile = cwd + '/tests.log' daiquiri.setup(level=logging.INFO, outputs=(daiquiri.output.File(logfile), )) def main(): return 0 if __name__ == "__main__": main()
import daiquiri import logging from itertools import groupby from datetime import datetime, timedelta from collections import namedtuple import pandas as pd from jitenshea import config from jitenshea.iodb import db daiquiri.setup(level=logging.INFO) logger = daiquiri.getLogger(__name__) CITIES = ('bordeaux', 'lyon') TimeWindow = namedtuple('TimeWindow', ['start', 'stop', 'order_reference_date']) def processing_daily_data(rset, window): """Re arrange when it's necessary the daily transactions data rset: ResultProxy by SQLAlchemy Result of a SQL query Return a list of dicts """
def setUpClass(self): super(TestCase, self).setUpClass() self.conf = service.prepare_service([], conf=utils.prepare_conf(), default_config_files=[], logging_level=logging.DEBUG) if not os.getenv("GNOCCHI_TEST_DEBUG"): daiquiri.setup(outputs=[]) py_root = os.path.abspath( os.path.join( os.path.dirname(__file__), '..', )) self.conf.set_override('paste_config', os.path.join(py_root, 'rest', 'api-paste.ini'), group="api") self.conf.set_override('policy_file', os.path.join(py_root, 'rest', 'policy.json'), group="oslo_policy") # NOTE(jd) This allows to test S3 on AWS if not os.getenv("AWS_ACCESS_KEY_ID"): self.conf.set_override('s3_endpoint_url', os.getenv("GNOCCHI_STORAGE_HTTP_URL"), group="storage") self.conf.set_override('s3_access_key_id', "gnocchi", group="storage") self.conf.set_override('s3_secret_access_key', "anythingworks", group="storage") self.index = indexer.get_driver(self.conf) # NOTE(jd) So, some driver, at least SQLAlchemy, can't create all # their tables in a single transaction even with the # checkfirst=True, so what we do here is we force the upgrade code # path to be sequential to avoid race conditions as the tests run # in parallel. self.coord = coordination.get_coordinator( self.conf.storage.coordination_url, str(uuid.uuid4()).encode('ascii')) self.coord.start(start_heart=True) with self.coord.get_lock(b"gnocchi-tests-db-lock"): self.index.upgrade() self.coord.stop() self.archive_policies = self.ARCHIVE_POLICIES.copy() for name, ap in six.iteritems(self.archive_policies): # Create basic archive policies try: self.index.create_archive_policy(ap) except indexer.ArchivePolicyAlreadyExists: pass storage_driver = os.getenv("GNOCCHI_TEST_STORAGE_DRIVER", "file") self.conf.set_override('driver', storage_driver, 'storage') if storage_driver == 'ceph': self.conf.set_override('ceph_conffile', os.getenv("CEPH_CONF"), 'storage')
E-mail: zhangsuofei at njupt.edu.cn | hangyu5 at illinois.edu """ import tensorflow as tf import tensorflow.contrib.slim as slim from config import cfg, get_coord_add, get_dataset_size_train, get_num_classes, get_create_inputs import time import numpy as np import sys import os import capsnet_em as net import logging import daiquiri daiquiri.setup(level=logging.DEBUG) logger = daiquiri.getLogger(__name__) def main(args): """Get dataset hyperparameters.""" assert len(args) == 2 and isinstance(args[1], str) dataset_name = args[1] logger.info('Using dataset: {}'.format(dataset_name)) """Set reproduciable random seed""" tf.set_random_seed(1234) coord_add = get_coord_add(dataset_name) dataset_size = get_dataset_size_train(dataset_name) num_classes = get_num_classes(dataset_name)
import logging import psycopg2 import daiquiri from psycopg2.extras import DictCursor from repo.app import REVIEW_DB_SETTINGS from review.compare import diffs from review.database import query_review_reports_old_metrics, update_metrics daiquiri.setup( level=logging.DEBUG, outputs=( daiquiri.output.File('poll-errors.log', level=logging.ERROR), daiquiri.output.RotatingFile( 'poll-debug.log', level=logging.DEBUG, # 10 MB max_size_bytes=10000000))) def get_review_db(): db = psycopg2.connect(**REVIEW_DB_SETTINGS) return db def _update_metrics(): before = {} before_list = [] after = {} after_list = [] review_db = get_review_db()