def build_config(config_dirs=DEFAULT_DIRS, logging_config="logging_config.yaml"): # Loads files in alphabetical order based on the bare filename config_file_paths = [] for directory in config_dirs: dir_glob = os.path.join(directory, "*.yaml") config_file_paths.extend( (os.path.basename(path), path) for path in glob.glob(dir_glob)) config_file_paths.sort() logger.info("Loading configs from {0}".format(config_file_paths)) config = yamlconf.load(*(open(p) for fn, p in config_file_paths)) # Load logging config if specified if logging_config is not None: with open(logging_config) as f: logging_config = yamlconf.load(f) logging.config.dictConfig(logging_config) # Add nltk data path if specified. if 'data_paths' in config['ores'] and \ 'nltk' in config['ores']['data_paths']: import nltk nltk.data.path.append(config['ores']['data_paths']['nltk']) return config
def configure_logging(verbose=False, debug=False, logging_config=None, **kwargs): # Load logging config if specified. If no config file is specified, we # make a half-hearted attempt to find a distributed logging_config.yaml # in the current working directory. if logging_config is None: if os.path.exists(DEFAULT_LOGGING_CONFIG): logging_config = DEFAULT_LOGGING_CONFIG if logging_config is not None: with open(logging_config) as f: logging_config = yamlconf.load(f) logging.config.dictConfig(logging_config) # Secret sauce: if running from the console, mirror logs there. if sys.stdin.isatty(): handler = logging.StreamHandler(stream=sys.stderr) formatter = logging.Formatter(fmt=DEFAULT_FORMAT) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) else: # Configure fallback logging. logging.basicConfig(level=logging.INFO, format=DEFAULT_FORMAT) if debug: logging.getLogger().setLevel(logging.DEBUG)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) config_paths = os.path.join(args['--config'], "*.yaml") config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths)))) processes = int(args['--processes']) verbose = args['--verbose'] debug = args['--debug'] logging.basicConfig( level=logging.DEBUG if debug else logging.INFO, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s') logging.getLogger('requests').setLevel(logging.INFO) if verbose: logging.getLogger('revscoring.dependencies.dependent') \ .setLevel(logging.DEBUG) else: logging.getLogger('revscoring.dependencies.dependent') \ .setLevel(logging.INFO) logging.getLogger("ores.metrics_collectors.logger").setLevel(logging.DEBUG) app = server.configure(config) app.run(host=args['--host'], port=int(args['--port']), debug=True, processes=processes)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) config_paths = os.path.join(args['--config'], "*.yaml") config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths)))) db = DB.from_config(config) run(db)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) logging.basicConfig( level=logging.INFO if not args['--debug'] else logging.DEBUG, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s') # Requests is loud. Be quiet requests. logging.getLogger("requests").setLevel(logging.WARNING) requests.packages.urllib3.disable_warnings() # If we're using logging for metrics collection, show it. logging.getLogger("ores.metrics_collectors").setLevel(logging.DEBUG) stream_url = args['<stream-url>'] ores_url = args['<ores-url>'] config_paths = os.path.join(args['--config'], "*.yaml") config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths)))) delay = float(args['--delay']) verbose = bool(args['--debug']) ss_name = config['ores']['scoring_system'] if 'metrics_collector' in config['scoring_systems'][ss_name]: metrics_collector = MetricsCollector.from_config( config, config['scoring_systems'][ss_name]['metrics_collector']) else: metrics_collector = Null() run(stream_url, ores_url, metrics_collector, config, delay, verbose)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) logging.basicConfig( level=logging.INFO if not args['--debug'] else logging.DEBUG, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s') logging.getLogger("revscoring.scoring.models").setLevel(logging.WARNING) params_config = yamlconf.load(open(args['<params-config>'])) features_path = args['<features>'] features = yamlconf.import_path(features_path) if args['--observations'] == "<stdin>": observations = read_observations(sys.stdin) else: observations = read_observations(open(args['--observations'])) logger.info("Reading feature values & labels...") label_name = args['<label>'] value_labels = \ [(list(solve(features, cache=ob['cache'])), ob[label_name]) for ob in observations] statistic_path = args['<statistic>'] additional_params = {} labels, label_weights, population_rates = \ util.read_labels_and_population_rates( None, args['--label-weight'], args['--pop-rate']) if label_weights is not None: additional_params['label_weights'] = label_weights if population_rates is not None: additional_params['population_rates'] = population_rates maximize = not args['--minimize'] folds = int(args['--folds']) if args['--report'] == "<stdout>": report = sys.stdout else: report = open(args['--report'], "w") if args['--processes'] == "<cpu-count>": processes = multiprocessing.cpu_count() else: processes = int(args['--processes']) if args['--cv-timeout'] == "<forever>": cv_timeout = None else: cv_timeout = float(args['--cv-timeout']) * 60 # Convert to seconds verbose = args['--verbose'] run(params_config, features, features_path, value_labels, statistic_path, additional_params, maximize, folds, report, processes, cv_timeout, verbose)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) # This expects the database config file path db_config = yamlconf.load(open(args['<config>'])) db = DB.from_params(**db_config) reload_test_data = args['--reload-test-data'] run(db, reload_test_data)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) config_paths = os.path.join(args['--config'], "*.yaml") config = yamlconf.load( *(open(p) for p in sorted(glob.glob(config_paths)))) db = DB.from_config(config) run(db)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) stream_url = args['<stream-url>'] ores_url = args['<ores-url>'] config = yamlconf.load(open(args['--config'])) delay = float(args['--delay']) verbose = bool(args['--verbose']) run(stream_url, ores_url, config, delay, verbose)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) campaign_id = int(args['<campaign-id>']) tasks = (json.loads(line) for line in sys.stdin) config_paths = os.path.join(args['--config'], "*.yaml") config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths)))) db = DB.from_config(config) run(db, campaign_id, tasks)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) diff_docs = read_docs(sys.stdin) session = api.Session(args['--api']) config_doc = yamlconf.load(open(args['--config'])) diff_engine = DiffEngine.from_config(config_doc, config_doc["diff_engine"]) run(diff_docs, session, diff_engine)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) stream_url = args['<stream-url>'] ores_url = args['<ores-url>'] config_paths = os.path.join(args['--config'], "*.yaml") config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths)))) delay = float(args['--delay']) verbose = bool(args['--verbose']) notify = notify_socket() run(stream_url, ores_url, config, delay, notify, verbose)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) # This expects the database config file path config_paths = os.path.join(args['--config'], "*.yaml") config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths)))) db = DB.from_config(config) reload_test_data = args['--reload-test-data'] run(db, reload_test_data)
def main(): config_path = os.path.dirname(os.path.abspath(__file__)) + '/config.yaml' with open(config_path, 'r') as f: config = yamlconf.load(f) data = get_ores_data(config['ores_host']) data = deep_merge.merge(data, get_wikilabels_data(config['wikilabels_host'])) data = {'data': data, 'timestamp': time.time()} path = os.path.dirname(os.path.abspath(__file__)) + '/static/data.json' with open(path, 'w') as f: f.write(json.dumps(data))
def main(): args = docopt.docopt(__doc__) logging.basicConfig( level=logging.INFO if not args['--debug'] else logging.DEBUG, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s') logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING) taxon_paths = args['<taxon>'] logger.info("Loading taxon from {0}".format(taxon_paths)) taxonomy = yamlconf.load(*(open(p) for p in taxon_paths)) return print_nodes(taxonomy)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) campaign_id = int(args['<campaign-id>']) tasks = (json.loads(line) for line in sys.stdin) config_paths = os.path.join(args['--config'], "*.yaml") config = yamlconf.load( *(open(p) for p in sorted(glob.glob(config_paths)))) db = DB.from_config(config) run(db, campaign_id, tasks)
def build_config(config_dirs=DEFAULT_DIRS, **kwargs): # Loads files in alphabetical order based on the bare filename config_file_paths = [] for directory in config_dirs: dir_glob = os.path.join(directory, "*.yaml") config_file_paths.extend( (os.path.basename(path), path) for path in glob.glob(dir_glob)) config_file_paths.sort() logger.info("Loading configs from {0}".format(config_file_paths)) config = yamlconf.load(*(open(p) for fn, p in config_file_paths)) return config
def main(argv=None): logging.basicConfig(level=logging.DEBUG) args = docopt.docopt(__doc__, argv=argv) config = yamlconf.load(open(args['--config'])) name = config['ores']['score_processor'] score_processor = Celery.from_config(config, name) score_processor.application.worker_main( argv=["celery_worker", "--loglevel=INFO"] )
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) if args['--config'] is not None: config = yamlconf.load(open(args['--config'])) else: config = None app = application.configure(config) app.run(host="0.0.0.0", port=int(args['--port']), debug=True, ssl_context="adhoc")
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) config_paths = os.path.join(args['--config'], "*.yaml") config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths)))) if args['--ssl']: kwargs = {'ssl_context': "adhoc"} else: kwargs = {} app = server.configure(config) app.debug = True app.run(host="0.0.0.0", port=int(args['--port']), debug=True, **kwargs)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) if args['--config'] is not None: config = yamlconf.load(open(args['--config'])) else: config = None app = server.configure(config) app.debug = True app.run(host="0.0.0.0", port=int(args['--port']), debug=True, ssl_context="adhoc")
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) logging.basicConfig( level=logging.INFO if not args['--debug'] else logging.DEBUG, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s' ) params_config = yamlconf.load(open(args['<params-config>'])) features_path = args['<features>'] features = yamlconf.import_path(features_path) if args['--observations'] == "<stdin>": observations = read_observations(sys.stdin) else: observations = read_observations(open(args['--observations'])) logger.info("Reading feature values & labels...") label_name = args['<label>'] value_labels = \ [(list(solve(features, cache=ob['cache'])), ob[label_name]) for ob in observations] # Get a sepecialized scorer if we have one scoring = metrics.SCORERS.get(args['--scoring'], args['--scoring']) folds = int(args['--folds']) if args['--report'] == "<stdout>": report = sys.stdout else: report = open(args['--report'], "w") if args['--processes'] == "<cpu-count>": processes = multiprocessing.cpu_count() else: processes = int(args['--processes']) if args['--cv-timeout'] == "<forever>": cv_timeout = None else: cv_timeout = float(args['--cv-timeout']) * 60 # Convert to seconds scale_features = args['--scale-features'] verbose = args['--verbose'] run(params_config, features_path, value_labels, scoring, folds, report, processes, cv_timeout, scale_features, verbose)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) wiki = args['<wiki>'] name = args['<name>'] form = args['<form>'] view = args['<view>'] labels_per_task = args['<labels-per-task>'] tasks_per_assignment = args['<tasks-per-assignment>'] config_paths = os.path.join(args['--config'], "*.yaml") config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths)))) db = DB.from_config(config) force = args['--force'] run(db, wiki, name, form, view, labels_per_task, tasks_per_assignment, force)
def main(): args = docopt.docopt(__doc__) logging.basicConfig( level=logging.INFO if not args['--debug'] else logging.DEBUG, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s') logging.getLogger("urllib3.connectionpool").setLevel(logging.WARNING) taxon_paths = args['<taxon>'] logger.info("Loading taxon from {0}".format(taxon_paths)) taxonomy = yamlconf.load(*(open(p) for p in taxon_paths)) session = mwapi.Session(ENWIKI_HOST, user_agent=args['--ua-email']) threads = int(args['--threads']) return check_wikiprojects(taxonomy, session, threads)
def index(): """Application landing page.""" SITE_ROOT = os.path.realpath(os.path.dirname(__file__)) json_url = os.path.join(SITE_ROOT, 'static', 'data.json') config_path = os.path.dirname(os.path.abspath(__file__)) + '/config.yaml' with open(config_path, 'r') as f: config = yamlconf.load(f) data = json.load(open(json_url)) wikis = transform_data(data['data']) update = time.strftime("%d %B %Y %H:%M:%S UTC", time.gmtime(data['timestamp'])) return flask.render_template('index.html', wikis=wikis, update=update, **config)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) config = yamlconf.load(open(args["--config"])) processes = int(args["--processes"]) logging.basicConfig( level=logging.INFO if not args["--verbose"] else logging.DEBUG, format="%(asctime)s %(levelname)s:%(name)s -- %(message)s", ) logging.getLogger("ores.metrics_collectors.logger").setLevel(logging.DEBUG) app = server.configure(config) app.run(host=args["--host"], port=int(args["--port"]), debug=True, processes=processes)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) config = yamlconf.load(open(args['--config'])) if args['--verbose']: logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s' ) app = server.configure(config) app.run(host="0.0.0.0", port=int(args['--port']), debug=True, threaded=True)
def main(argv=None): args = docopt(__doc__, argv=argv) weights = yamlconf.load(open(args['<weights>'])) if args['--scores'] == '<stdin>': revision_scores = read_revision_scores(sys.stdin) else: revision_scores = read_revision_scores(open(args['--scores'])) if args['--output'] == '<stdout>': output = sys.stdout else: output = open(args['--output']) run(revision_scores, weights, output)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) config_doc = yamlconf.load(open(args['--config'])) diff_engine = DiffEngine.from_config(config_doc, config_doc["diff_engine"]) drop_text = bool(args['--drop-text']) if args['--timeout'] == "<infinity>": timeout = None else: timeout = float(args['--timeout']) verbose = bool(args['--verbose']) run(read_docs(sys.stdin), diff_engine, timeout, drop_text, verbose)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) logging.basicConfig( level=logging.INFO if not args['--debug'] else logging.DEBUG, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s' ) params_config = yamlconf.load(open(args['<params-config>'])) features_path = args['<features>'] features = yamlconf.import_module(features_path) label_decoder = util.DECODERS[args['--label-type']] if args['--observations'] == "<stdin>": observations_f = sys.stdin else: observations_f = open(args['--observations']) observations = util.read_observations(observations_f, features, label_decoder) # Get a sepecialized scorer if we have one scoring = metrics.SCORERS.get(args['--scoring'], args['--scoring']) folds = int(args['--folds']) if args['--report'] == "<stdout>": report = sys.stdout else: report = open(args['--report'], "w") if args['--processes'] == "<cpu-count>": processes = multiprocessing.cpu_count() else: processes = int(args['--processes']) if args['--cv-timeout'] == "<forever>": cv_timeout = None else: cv_timeout = float(args['--cv-timeout']) * 60 # Convert to seconds scale_features = args['--scale-features'] verbose = args['--verbose'] run(params_config, features_path, observations, scoring, folds, report, processes, cv_timeout, scale_features, verbose)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) logging.basicConfig( level=logging.INFO, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s') ores_level = logging.DEBUG if args['--verbose'] else logging.INFO logging.getLogger('ores').setLevel(ores_level) config_paths = os.path.join(args['--config'], "*.yaml") config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths)))) name = config['ores']['score_processor'] score_processor = Celery.from_config(config, name) score_processor.application.worker_main( argv=["celery_worker", "--loglevel=INFO"])
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) logging.basicConfig( level=logging.INFO if not args['--debug'] else logging.DEBUG, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s') params_config = yamlconf.load(open(args['<params-config>'])) features_path = args['<features>'] features = yamlconf.import_module(features_path) label_decoder = util.DECODERS[args['--label-type']] if args['--observations'] == "<stdin>": observations_f = sys.stdin else: observations_f = open(args['--observations']) observations = util.read_observations(observations_f, features, label_decoder) # Get a sepecialized scorer if we have one scoring = metrics.SCORERS.get(args['--scoring'], args['--scoring']) folds = int(args['--folds']) if args['--report'] == "<stdout>": report = sys.stdout else: report = open(args['--report'], "w") if args['--processes'] == "<cpu-count>": processes = multiprocessing.cpu_count() else: processes = int(args['--processes']) if args['--cv-timeout'] == "<forever>": cv_timeout = None else: cv_timeout = float(args['--cv-timeout']) * 60 # Convert to seconds scale_features = args['--scale-features'] verbose = args['--verbose'] run(params_config, features_path, observations, scoring, folds, report, processes, cv_timeout, scale_features, verbose)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) logging.basicConfig( level=logging.INFO, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s' ) ores_level = logging.DEBUG if args['--verbose'] else logging.INFO logging.getLogger('ores').setLevel(ores_level) config = yamlconf.load(open(args['--config'])) name = config['ores']['score_processor'] score_processor = Celery.from_config(config, name) score_processor.application.worker_main( argv=["celery_worker", "--loglevel=INFO"] )
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) if args['--config'] is not None: config = yamlconf.load(open(args['--config'])) else: config = None if args['--ssl']: kwargs = {'ssl_context': "adhoc"} else: kwargs = {} app = server.configure(config) app.debug = True app.run(host="0.0.0.0", port=int(args['--port']), debug=True, **kwargs)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) stream_url = args['<stream-url>'] ores_url = args['<ores-url>'] config_paths = os.path.join(args['--config'], "*.yaml") config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths)))) delay = float(args['--delay']) verbose = bool(args['--verbose']) notify = notify_socket() ss_name = config['ores']['scoring_system'] if 'metrics_collector' in config['scoring_systems'][ss_name]: metrics_collector = MetricsCollector.from_config( config, config['scoring_systems'][ss_name]['metrics_collector']) else: metrics_collector = Null() run(stream_url, ores_url, metrics_collector, config, delay, notify, verbose)
def read_labels_and_population_rates(labels_str, label_weights_strs, pop_rates_strs, config_path): # First try config file if config_path: labels_config = yamlconf.load(open(os.path.expanduser(config_path))) return read_labels_config(labels_config) # Try to read --labels if labels_str is not None: labels = [json.loads(l) for l in labels_str.strip().split(",")] else: labels = None # Try to read --label-weight if len(label_weights_strs) > 0: label_weights = OrderedDict() for label_weights_str in label_weights_strs: label, weight = ( json.loads(s) for s in label_weights_str.split("=", 1)) label_weights[label] = weight else: label_weights = None # Try to read --pop-rate if len(pop_rates_strs) == 0: population_rates = None else: population_rates = OrderedDict() for label_rate_str in pop_rates_strs: label, rate = (json.loads(s) for s in label_rate_str.split("=", 1)) population_rates[label] = rate if labels is None and label_weights is None and population_rates is None: raise RuntimeError("Either --pop-rates or --labels or \ --labels-config must be specified") elif labels is None: if population_rates is not None: labels = list(population_rates.keys()) else: labels = list(label_weights.keys()) return labels, label_weights, population_rates
def read_labels_and_population_rates(labels_str, label_weights_strs, pop_rates_strs, config_path): # First try config file if config_path: labels_config = yamlconf.load(open(os.path.expanduser(config_path))) return read_labels_config(labels_config) # Try to read --labels if labels_str is not None: labels = [json.loads(l) for l in labels_str.strip().split(",")] else: labels = None # Try to read --label-weight if len(label_weights_strs) > 0: label_weights = OrderedDict() for label_weights_str in label_weights_strs: label, weight = (json.loads(s) for s in label_weights_str.split("=", 1)) label_weights[label] = weight else: label_weights = None # Try to read --pop-rate if len(pop_rates_strs) == 0: population_rates = None else: population_rates = OrderedDict() for label_rate_str in pop_rates_strs: label, rate = (json.loads(s) for s in label_rate_str.split("=", 1)) population_rates[label] = rate if labels is None and label_weights is None and population_rates is None: raise RuntimeError("Either --pop-rates or --labels or \ --labels-config must be specified") elif labels is None: if population_rates is not None: labels = list(population_rates.keys()) else: labels = list(label_weights.keys()) return labels, label_weights, population_rates
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) config_doc = yamlconf.load(open(args['--config'])) diff_engine = DiffEngine.from_config(config_doc, config_doc["diff_engine"]) drop_text = bool(args['--drop-text']) if args['--timeout'] == "<infinity>": timeout = None else: timeout = float(args['--timeout']) if args['--namespaces'] == "<all>": namespaces = None else: namespaces = set(int(ns) for ns in args['--namespaces'].split(",")) verbose = bool(args['--verbose']) run(read_docs(sys.stdin), diff_engine, timeout, namespaces, drop_text, verbose)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) if len(args['<dump_file>']) == 0: dump_files = [] else: dump_files = args['<dump_file>'] config_doc = yamlconf.load(open(args['--config'])) diff_engine = DiffEngine.from_config(config_doc, config_doc['diff_engine']) drop_text = bool(args['--drop-text']) if args['--threads'] == "<cpu_count>": threads = cpu_count() else: threads = int(args['--threads']) verbose = bool(args['--verbose']) run(dump_files, diff_engine, threads, drop_text, verbose)
def configure_logging(verbose=False, debug=False, logging_config=None, **kwargs): # Load logging config if specified. If no config file is specified, we # make a half-hearted attempt to find a distributed logging_config.yaml # in the current working directory. if logging_config is None: if os.path.exists(DEFAULT_LOGGING_CONFIG): logging_config = DEFAULT_LOGGING_CONFIG if logging_config is not None: with open(logging_config) as f: logging_config = yamlconf.load(f) logging.config.dictConfig(logging_config) # Secret sauce: if running from the console, mirror logs there. if sys.stdin.isatty(): handler = logging.StreamHandler(stream=sys.stderr) formatter = logging.Formatter(fmt=DEFAULT_FORMAT) handler.setFormatter(formatter) logging.getLogger().addHandler(handler) else: # Configure fallback logging. logging.basicConfig(level=logging.INFO, format=DEFAULT_FORMAT) logging.getLogger('requests').setLevel(logging.INFO) logging.getLogger('stopit').setLevel(logging.ERROR) # Command-line options can override some of the logging config, regardless # of whether logging_config.yaml was provided. # TODO: Document and reconcile debug vs verbose. if debug: logging.getLogger().setLevel(logging.DEBUG) logging.getLogger('ores.metrics_collectors.logger') \ .setLevel(logging.DEBUG) if verbose: logging.getLogger('revscoring.dependencies.dependent') \ .setLevel(logging.DEBUG)
#!/usr/bin/env python3 import glob import logging import logging.config from itertools import chain import yamlconf from wikilabels.wsgi import server config_paths = sorted(glob.glob("config/*.yaml")) if __name__ == '__main__': config_paths += sorted(glob.glob("config/localhost/*.yaml")) config = yamlconf.load(*(open(p) for p in config_paths)) application = server.configure(config) with open("logging_config.yaml") as f: logging_config = yamlconf.load(f) logging.config.dictConfig(logging_config) logging.basicConfig( level=logging.INFO, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s' ) if __name__ == '__main__': logging.getLogger('wikilabels').setLevel(logging.DEBUG) application.debug = True application.run(host="0.0.0.0", port=8080, debug=True)
def app(): config_paths = os.path.join('config/', "*.yaml") config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths)))) return server.configure(config)
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) config = yamlconf.load(open(args["--config"])) run(args["<stream-url>"], args["<ores-url>"], config, args["--verbose"])
def main(argv=None): args = docopt.docopt(__doc__, argv=argv) logging.basicConfig( level=logging.INFO if not args['--debug'] else logging.DEBUG, format='%(asctime)s %(levelname)s:%(name)s -- %(message)s' ) logging.getLogger("revscoring.scoring.models").setLevel(logging.WARNING) params_config = yamlconf.load(open(args['<params-config>'])) features_path = args['<features>'] features = yamlconf.import_path(features_path) if args['--observations'] == "<stdin>": observations = read_observations(sys.stdin) else: observations = read_observations(open(args['--observations'])) logger.info("Reading feature values & labels...") label_name = args['<label>'] value_labels = \ [(list(solve(features, cache=ob['cache'])), ob[label_name]) for ob in observations] statistic_path = args['<statistic>'] additional_params = {} labels, label_weights, population_rates = \ util.read_labels_and_population_rates( args['--labels'], args['--label-weight'], args['--pop-rate'], args['--labels-config']) if label_weights is not None: additional_params['label_weights'] = label_weights if population_rates is not None: additional_params['population_rates'] = population_rates if args['--center']: additional_params['center'] = args['--center'] if args['--scale']: additional_params['scale'] = args['--scale'], if args['--multilabel']: additional_params['multilabel'] = True maximize = not args['--minimize'] folds = int(args['--folds']) if args['--report'] == "<stdout>": report = sys.stdout else: report = open(args['--report'], "w") if args['--processes'] == "<cpu-count>": processes = multiprocessing.cpu_count() else: processes = int(args['--processes']) if args['--cv-timeout'] == "<forever>": cv_timeout = None else: cv_timeout = float(args['--cv-timeout']) * 60 # Convert to seconds verbose = args['--verbose'] run(params_config, features, labels, features_path, value_labels, statistic_path, additional_params, maximize, folds, report, processes, cv_timeout, verbose)
#!/usr/bin/env python3 import glob import logging import yamlconf from wikilabels.wsgi import server config = yamlconf.load(*(open(p) for p in sorted(glob.glob("config/*.yaml")))) application = server.configure(config) if __name__ == '__main__': logging.getLogger('wikilabels').setLevel(logging.DEBUG) application.debug = True application.run(host="0.0.0.0", debug=True)
#!/usr/bin/env python3 import os from flask import request import yamlconf from wikilabels.wsgi import application directory = os.path.dirname(os.path.realpath(__file__)) config_path = os.path.join(directory, "config/wikilabels.yaml") config = yamlconf.load(open(config_path)) app = application.configure(config) app.debug = True if __name__ == '__main__': app.run(host="0.0.0.0", debug = True)