def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)
    # This expects the database config file path
    db_config = yamlconf.load(open(args['<config>']))
    db = DB.from_params(**db_config)
    reload_test_data = args['--reload-test-data']

    run(db, reload_test_data)
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    config_paths = os.path.join(args['--config'], "*.yaml")
    config = yamlconf.load(
        *(open(p) for p in sorted(glob.glob(config_paths))))
    db = DB.from_config(config)
    run(db)
Beispiel #3
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    stream_url = args['<stream-url>']
    ores_url = args['<ores-url>']
    config = yamlconf.load(open(args['--config']))
    delay = float(args['--delay'])
    verbose = bool(args['--verbose'])
    run(stream_url, ores_url, config, delay,
        verbose)
Beispiel #4
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)
    # This expects the database config file path

    config_paths = os.path.join(args['--config'], "*.yaml")
    config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths))))

    db = DB.from_config(config)
    reload_test_data = args['--reload-test-data']

    run(db, reload_test_data)
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    diff_docs = read_docs(sys.stdin)

    session = api.Session(args['--api'])

    config_doc = yamlconf.load(open(args['--config']))
    diff_engine = DiffEngine.from_config(config_doc, config_doc["diff_engine"])

    run(diff_docs, session, diff_engine)
Beispiel #6
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    campaign_id = int(args['<campaign-id>'])

    tasks = (json.loads(line) for line in sys.stdin)

    config_paths = os.path.join(args['--config'], "*.yaml")
    config = yamlconf.load(
        *(open(p) for p in sorted(glob.glob(config_paths))))
    db = DB.from_config(config)
    run(db, campaign_id, tasks)
Beispiel #7
0
def main(argv=None):
    logging.basicConfig(level=logging.DEBUG)
    args = docopt.docopt(__doc__, argv=argv)

    config = yamlconf.load(open(args['--config']))

    name = config['ores']['score_processor']
    score_processor = Celery.from_config(config, name)

    score_processor.application.worker_main(
        argv=["celery_worker", "--loglevel=INFO"]
    )
Beispiel #8
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    if args['--config'] is not None:
        config = yamlconf.load(open(args['--config']))
    else:
        config = None

    app = application.configure(config)
    app.run(host="0.0.0.0",
            port=int(args['--port']),
            debug=True,
            ssl_context="adhoc")
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    stream_url = args['<stream-url>']
    ores_url = args['<ores-url>']
    config_paths = os.path.join(args['--config'], "*.yaml")
    config = yamlconf.load(*(open(p) for p in
                             sorted(glob.glob(config_paths))))
    delay = float(args['--delay'])
    verbose = bool(args['--verbose'])
    notify = notify_socket()
    run(stream_url, ores_url, config, delay,
        notify, verbose)
Beispiel #10
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    config = yamlconf.load(open(args["--config"]))

    processes = int(args["--processes"])

    logging.basicConfig(
        level=logging.INFO if not args["--verbose"] else logging.DEBUG,
        format="%(asctime)s %(levelname)s:%(name)s -- %(message)s",
    )

    logging.getLogger("ores.metrics_collectors.logger").setLevel(logging.DEBUG)

    app = server.configure(config)
    app.run(host=args["--host"], port=int(args["--port"]), debug=True, processes=processes)
Beispiel #11
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    wiki = args['<wiki>']
    name = args['<name>']
    form = args['<form>']
    view = args['<view>']
    labels_per_task = args['<labels-per-task>']
    tasks_per_assignment = args['<tasks-per-assignment>']
    config_paths = os.path.join(args['--config'], "*.yaml")
    config = yamlconf.load(*(open(p) for p in
                             sorted(glob.glob(config_paths))))
    db = DB.from_config(config)
    force = args['--force']
    run(db, wiki, name, form, view, labels_per_task, tasks_per_assignment,
        force)
Beispiel #12
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    config = yamlconf.load(open(args['--config']))

    if args['--verbose']:
        logging.basicConfig(
            level=logging.DEBUG,
            format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
        )

    app = server.configure(config)
    app.run(host="0.0.0.0",
            port=int(args['--port']),
            debug=True,
            threaded=True)
Beispiel #13
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    logging.basicConfig(
        level=logging.INFO if not args['--debug'] else logging.DEBUG,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )

    params_config = yamlconf.load(open(args['<params-config>']))

    features_path = args['<features>']
    features = yamlconf.import_module(features_path)

    label_decoder = util.DECODERS[args['--label-type']]
    if args['--observations'] == "<stdin>":
        observations_f = sys.stdin
    else:
        observations_f = open(args['--observations'])

    observations = util.read_observations(observations_f, features,
                                          label_decoder)

    # Get a sepecialized scorer if we have one
    scoring = metrics.SCORERS.get(args['--scoring'], args['--scoring'])

    folds = int(args['--folds'])

    if args['--report'] == "<stdout>":
        report = sys.stdout
    else:
        report = open(args['--report'], "w")

    if args['--processes'] == "<cpu-count>":
        processes = multiprocessing.cpu_count()
    else:
        processes = int(args['--processes'])

    if args['--cv-timeout'] == "<forever>":
        cv_timeout = None
    else:
        cv_timeout = float(args['--cv-timeout']) * 60  # Convert to seconds

    scale_features = args['--scale-features']
    verbose = args['--verbose']

    run(params_config, features_path, observations, scoring, folds,
        report, processes, cv_timeout, scale_features, verbose)
Beispiel #14
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    config_paths = os.path.join(args['--config'], "*.yaml")
    config = yamlconf.load(*(open(p) for p in sorted(glob.glob(config_paths))))

    if args['--ssl']:
        kwargs = {'ssl_context': "adhoc"}
    else:
        kwargs = {}

    app = server.configure(config)
    app.debug = True
    app.run(host="0.0.0.0",
            port=int(args['--port']),
            debug=True,
            **kwargs)
Beispiel #15
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )
    ores_level = logging.DEBUG if args['--verbose'] else logging.INFO
    logging.getLogger('ores').setLevel(ores_level)

    config = yamlconf.load(open(args['--config']))

    name = config['ores']['score_processor']
    score_processor = Celery.from_config(config, name)

    score_processor.application.worker_main(
        argv=["celery_worker", "--loglevel=INFO"]
    )
Beispiel #16
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    if args['--config'] is not None:
        config = yamlconf.load(open(args['--config']))
    else:
        config = None

    if args['--ssl']:
        kwargs = {'ssl_context': "adhoc"}
    else:
        kwargs = {}

    app = server.configure(config)
    app.debug = True
    app.run(host="0.0.0.0",
            port=int(args['--port']),
            debug=True,
            **kwargs)
Beispiel #17
0
def read_labels_and_population_rates(labels_str, label_weights_strs,
                                     pop_rates_strs, config_path):
    # First try config file
    if config_path:
        labels_config = yamlconf.load(open(os.path.expanduser(config_path)))
        return read_labels_config(labels_config)

    # Try to read --labels
    if labels_str is not None:
        labels = [json.loads(l) for l in labels_str.strip().split(",")]
    else:
        labels = None

    # Try to read --label-weight
    if len(label_weights_strs) > 0:
        label_weights = OrderedDict()
        for label_weights_str in label_weights_strs:
            label, weight = (
                json.loads(s) for s in label_weights_str.split("=", 1))
            label_weights[label] = weight
    else:
        label_weights = None

    # Try to read --pop-rate
    if len(pop_rates_strs) == 0:
        population_rates = None
    else:
        population_rates = OrderedDict()
        for label_rate_str in pop_rates_strs:
            label, rate = (json.loads(s) for s in label_rate_str.split("=", 1))
            population_rates[label] = rate

    if labels is None and label_weights is None and population_rates is None:
        raise RuntimeError("Either --pop-rates or --labels or \
                           --labels-config must be specified")
    elif labels is None:
        if population_rates is not None:
            labels = list(population_rates.keys())
        else:
            labels = list(label_weights.keys())

    return labels, label_weights, population_rates
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    config_doc = yamlconf.load(open(args['--config']))
    diff_engine = DiffEngine.from_config(config_doc, config_doc["diff_engine"])

    drop_text = bool(args['--drop-text'])

    if args['--timeout'] == "<infinity>":
        timeout = None
    else:
        timeout = float(args['--timeout'])

    if args['--namespaces'] == "<all>":
        namespaces = None
    else:
        namespaces = set(int(ns) for ns in args['--namespaces'].split(","))

    verbose = bool(args['--verbose'])

    run(read_docs(sys.stdin), diff_engine, timeout, namespaces, drop_text, verbose)
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    if len(args['<dump_file>']) == 0:
        dump_files = []
    else:
        dump_files = args['<dump_file>']

    config_doc = yamlconf.load(open(args['--config']))
    diff_engine = DiffEngine.from_config(config_doc, config_doc['diff_engine'])

    drop_text = bool(args['--drop-text'])

    if args['--threads'] == "<cpu_count>":
        threads = cpu_count()
    else:
        threads = int(args['--threads'])

    verbose = bool(args['--verbose'])

    run(dump_files, diff_engine, threads, drop_text, verbose)
#!/usr/bin/env python3
import glob
import logging

import yamlconf
from wikilabels.wsgi import server

config = yamlconf.load(*(open(p) for p in sorted(glob.glob("config/*.yaml"))))

application = server.configure(config)


if __name__ == '__main__':
    logging.getLogger('wikilabels').setLevel(logging.DEBUG)

    application.debug = True
    application.run(host="0.0.0.0", debug=True)
Beispiel #21
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    logging.basicConfig(
        level=logging.INFO if not args['--debug'] else logging.DEBUG,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )
    logging.getLogger("revscoring.scoring.models").setLevel(logging.WARNING)

    params_config = yamlconf.load(open(args['<params-config>']))

    features_path = args['<features>']
    features = yamlconf.import_path(features_path)

    if args['--observations'] == "<stdin>":
        observations = read_observations(sys.stdin)
    else:
        observations = read_observations(open(args['--observations']))

    logger.info("Reading feature values & labels...")
    label_name = args['<label>']
    value_labels = \
        [(list(solve(features, cache=ob['cache'])), ob[label_name])
         for ob in observations]

    statistic_path = args['<statistic>']
    additional_params = {}

    labels, label_weights, population_rates = \
        util.read_labels_and_population_rates(
            args['--labels'], args['--label-weight'], args['--pop-rate'],
            args['--labels-config'])
    if label_weights is not None:
        additional_params['label_weights'] = label_weights
    if population_rates is not None:
        additional_params['population_rates'] = population_rates

    if args['--center']:
        additional_params['center'] = args['--center']
    if args['--scale']:
        additional_params['scale'] = args['--scale'],

    if args['--multilabel']:
        additional_params['multilabel'] = True

    maximize = not args['--minimize']

    folds = int(args['--folds'])

    if args['--report'] == "<stdout>":
        report = sys.stdout
    else:
        report = open(args['--report'], "w")

    if args['--processes'] == "<cpu-count>":
        processes = multiprocessing.cpu_count()
    else:
        processes = int(args['--processes'])

    if args['--cv-timeout'] == "<forever>":
        cv_timeout = None
    else:
        cv_timeout = float(args['--cv-timeout']) * 60  # Convert to seconds

    verbose = args['--verbose']

    run(params_config, features, labels, features_path, value_labels,
        statistic_path, additional_params, maximize, folds, report,
        processes, cv_timeout, verbose)
Beispiel #22
0
def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    config = yamlconf.load(open(args["--config"]))
    run(args["<stream-url>"], args["<ores-url>"], config, args["--verbose"])
Beispiel #23
0
#!/usr/bin/env python3
import os

from flask import request

import yamlconf
from wikilabels.wsgi import application

directory = os.path.dirname(os.path.realpath(__file__))

config_path = os.path.join(directory, "config/wikilabels.yaml")

config = yamlconf.load(open(config_path))

app = application.configure(config)
app.debug = True


if __name__ == '__main__':
    app.run(host="0.0.0.0", debug = True)
#!/usr/bin/env python3
import glob
import logging
import logging.config
from itertools import chain

import yamlconf
from wikilabels.wsgi import server

config_paths = sorted(glob.glob("config/*.yaml"))
if __name__ == '__main__':
    config_paths += sorted(glob.glob("config/localhost/*.yaml"))

config = yamlconf.load(*(open(p) for p in config_paths))
application = server.configure(config)

with open("logging_config.yaml") as f:
    logging_config = yamlconf.load(f)
    logging.config.dictConfig(logging_config)

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
)

if __name__ == '__main__':
    logging.getLogger('wikilabels').setLevel(logging.DEBUG)

    application.debug = True
    application.run(host="0.0.0.0", port=8080, debug=True)