Example #1
0
def import_data(archive):
    """Imports data dump into the database."""

    # inits the db engine
    create_app()
    print('Importing data...')
    db.dump.import_db_dump(archive)
Example #2
0
def compute_stats():
    """Compute any outstanding hourly stats and add to the database."""
    # inits the db engine
    create_app()
    import datetime
    import pytz
    db.stats.compute_stats(datetime.datetime.now(pytz.utc))
Example #3
0
def generate_snapshots():
    create_app()
    with db.engine.connect() as connection:
        for job in get_all_jobs(connection):
            set_snapshot_id(
                connection=connection,
                job_id=job["job_id"],
                snapshot_id=db.dataset.create_snapshot(job["dataset_id"])
            )
Example #4
0
def add_admin(username, force=False):
    """Make user an admin."""
    # inits the db engine
    create_app()
    try:
        db.user.set_admin(username, admin=True, force=force)
    except db.exceptions.DatabaseException as e:
        click.echo("Error: %s" % e, err=True)
    click.echo("Made %s an admin." % username)
Example #5
0
def remove_admin(username):
    """Remove admin privileges from a user."""
    # inits the db engine
    create_app()
    try:
        db.user.set_admin(username, admin=False)
    except db.exceptions.DatabaseException as e:
        click.echo("Error: %s" % e, err=True)
    click.echo("Removed admin privileges from %s." % username)
Example #6
0
def init_db(archive, force):
    """Initializes database and imports data if needed.

    This process involves several steps:
    1. Table structure is created.
    2. Data is imported from the archive if it is specified.
    3. Primary keys and foreign keys are created.
    4. Indexes are created.

    Data dump needs to be a .tar.xz archive produced by export command.

    More information about populating a PostgreSQL database efficiently can be
    found at http://www.postgresql.org/docs/current/static/populate.html.
    """
    if force:
        exit_code = _run_psql('drop_db.sql')
        if exit_code != 0:
            raise Exception(
                'Failed to drop existing database and user! Exit code: %i' %
                exit_code)

    print('Creating user and a database...')
    exit_code = _run_psql('create_db.sql')
    if exit_code != 0:
        raise Exception(
            'Failed to create new database and user! Exit code: %i' %
            exit_code)

    print('Creating database extensions...')
    exit_code = _run_psql('create_extensions.sql', 'acousticbrainz')
    if exit_code != 0:
        raise Exception('Failed to create database extensions! Exit code: %i' %
                        exit_code)

    # inits the db engine
    create_app()

    print('Creating types...')
    db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_types.sql'))

    print('Creating tables...')
    db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))

    if archive:
        print('Importing data...')
        db.dump.import_db_dump(archive)
    else:
        print('Skipping data importing.')

    print('Creating primary and foreign keys...')
    db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
    db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))

    print('Creating indexes...')
    db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))

    print("Done!")
Example #7
0
def incremental(location, id, threads):
    # inits the db engine
    create_app()
    dump_id, start_t, end_t = dump.prepare_incremental_dump(
        int(id) if id else None)
    print("Creating incremental dumps with data between %s and %s:\n" %
          (start_t, end_t))
    _incremental_db(location, dump_id, threads)
    _incremental_json_lowlevel(location, dump_id)
    _incremental_json_highlevel(location, dump_id)
Example #8
0
def json(location, rotate, no_lowlevel, no_highlevel):
    # inits the db engine
    create_app()
    if no_lowlevel and no_highlevel:
        print("wut? check your options, mate!")

    if not no_lowlevel:
        _json_lowlevel(location, rotate)

    if not no_highlevel:
        _json_highlevel(location, rotate)
Example #9
0
def full_db(location, threads, rotate):
    # inits the db engine
    create_app()
    print("Creating full database dump...")
    path = dump.dump_db(location, threads)
    print("Done! Created:", path)

    if rotate:
        print("Removing old dumps (except two latest)...")
        remove_old_archives(location,
                            "acousticbrainz-dump-[0-9]+-[0-9]+.tar.xz",
                            is_dir=False,
                            sort_key=lambda x: os.path.getmtime(x))
Example #10
0
def init_db(force, skip_create):
    """Initializes database.

    This process involves several steps:
    1. Table structure is created.
    2. Primary keys and foreign keys are created.
    3. Indexes are created.
    """

    uri = urlsplit(create_app().config['SQLALCHEMY_DATABASE_URI'])
    if force:
        exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER +
                                    ' -h ' + uri.hostname + ' -p ' + str(uri.port) + ' < ' +
                                    os.path.join(ADMIN_SQL_DIR, 'drop_db.sql'),
                                    shell=True)
        if exit_code != 0:
            raise Exception('Failed to drop existing database and user! Exit code: %i' % exit_code)


    if not skip_create:
        print('Creating user and a database...')
        exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER +
                                    ' -h ' + uri.hostname + ' -p ' + str(uri.port) + ' < ' +
                                    os.path.join(ADMIN_SQL_DIR, 'create_db.sql'),
                                    shell=True)
        if exit_code != 0:
            raise Exception('Failed to create new database and user! Exit code: %i' % exit_code)

        print('Creating database extensions...')
        exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' -d listenbrainz ' +
                                    '-h ' + uri.hostname + ' -p ' + str(uri.port) + ' < ' +
                                    os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'),
                                    shell=True)
        if exit_code != 0:
            raise Exception('Failed to create database extensions! Exit code: %i' % exit_code)

    with create_app().app_context():
        print('Creating tables...')
        db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))

        print('Creating primary and foreign keys...')
        db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
        db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))

        print('Creating indexes...')
        db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))

    print("Done!")
Example #11
0
def incremental_info(all=False):
    """Prints information about incremental dumps: id, timestamp.

    By default outputs information for the latest dump.
    """
    # inits the db engine
    create_app()
    info = dump.list_incremental_dumps()
    if info:
        if all:
            print('Incremental dumps:')
            for current in info:
                print(' - %s at %s' % current)
        else:
            print('Last dump ID: %s\nTimestamp: %s' % info[0])
    else:
        print('No incremental dumps yet.')
Example #12
0
def runserver(host, port, debug=False):
    application = webserver.create_app()
    webserver.schedule_jobs(application)
    run_simple(hostname=host,
               port=port,
               application=application,
               use_debugger=debug,
               use_reloader=debug,
               processes=5)
    def test_flask_debugtoolbar(self):
        """ Test if flask debugtoolbar is loaded correctly

        Creating an app with default config so that debug is True
        and SECRET_KEY is defined.
        """
        app = create_app(debug=True)
        client = app.test_client()
        resp = client.get(url_for('index.goals'))
        self.assert200(resp)
        self.assertIn('flDebug', str(resp.data))
def clear_memcached():
    import sys
    print(sys.path)
    app = create_app()
    try:
        cache.init(app.config["MEMCACHED_SERVERS"])
        cache.flush_all()
        print(green("Flushed everything from memcached.", bold=True))
    except AttributeError as e:
        print(
            red("Failed to clear memcached! Check your config file.\nError: %s"
                % e))
Example #15
0
def init_db(force, create_db):
    """Initializes database.

    This process involves several steps:
    1. Table structure is created.
    2. Primary keys and foreign keys are created.
    3. Indexes are created.
    """

    db.init_db_connection(config.POSTGRES_ADMIN_URI)
    if force:
        res = db.run_sql_script_without_transaction(
            os.path.join(ADMIN_SQL_DIR, 'drop_db.sql'))
        if not res:
            raise Exception(
                'Failed to drop existing database and user! Exit code: %i' %
                res)

    if create_db:
        print('Creating user and a database...')
        res = db.run_sql_script_without_transaction(
            os.path.join(ADMIN_SQL_DIR, 'create_db.sql'))
        if not res:
            raise Exception(
                'Failed to create new database and user! Exit code: %i' % res)

        print('Creating database extensions...')
        res = db.run_sql_script_without_transaction(
            os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'))
    # Don't raise an exception if the extension already exists

    application = webserver.create_app()
    with application.app_context():
        print('Creating schema...')
        db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_schema.sql'))

        print('Creating tables...')
        db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))

        print('Creating primary and foreign keys...')
        db.run_sql_script(
            os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
        db.run_sql_script(
            os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))

        print('Creating indexes...')
        db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))

    print('Create influx database...')
    subprocess.call(os.path.join(ADMIN_INFLUX_DIR, 'create_db.py'))
    print("Done!")
Example #16
0
def main():
    logging.info("Starting dataset evaluator...")
    app = create_app()
    dataset_dir = app.config["DATASET_DIR"]
    storage_dir = os.path.join(app.config["FILE_STORAGE_DIR"], "history")
    while True:
        pending_job = db.dataset_eval.get_next_pending_job()
        if pending_job:
            logging.info("Processing job %s..." % pending_job["id"])
            evaluate_dataset(pending_job, dataset_dir, storage_dir)
        else:
            logging.info("No pending datasets. Sleeping %s seconds." %
                         SLEEP_DURATION)
            time.sleep(SLEEP_DURATION)
Example #17
0
def full(ctx, location, threads, rotate):
    """This command creates:
    1. New incremental dump record (+ incremental dumps unless it's the first record)
    2. Full database dump
    3. Full JSON dump

    Archive rotation is enabled by default for full dumps.
    """
    # inits the db engine
    create_app()
    try:
        start_t = dump.prepare_incremental_dump()[1]
        if start_t:  # not the first incremental dump
            incremental(location, id=None, threads=None)
        else:
            print(
                "Skipping incremental dump creation since it's the first one.\n"
            )
    except dump.NoNewData:
        print("Skipping incremental dump creation. No new data.\n")

    ctx.invoke(full_db, location=location, threads=threads, rotate=rotate)
    ctx.invoke(json, location=location, rotate=rotate)
Example #18
0
def init_db(force):
    """Initializes database.

    This process involves several steps:
    1. Table structure is created.
    2. Primary keys and foreign keys are created.
    3. Indexes are created.
    """
    if force:
        exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' < ' +
                                    os.path.join(ADMIN_SQL_DIR, 'drop_db.sql'),
                                    shell=True)
        if exit_code != 0:
            raise Exception('Failed to drop existing database and user! Exit code: %i' % exit_code)

    print('Creating user and a database...')
    exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' < ' +
                                os.path.join(ADMIN_SQL_DIR, 'create_db.sql'),
                                shell=True)
    if exit_code != 0:
        raise Exception('Failed to create new database and user! Exit code: %i' % exit_code)

    print('Creating database extensions...')
    exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' -d messybrainz < ' +
                                os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'),
                                shell=True)
    if exit_code != 0:
        raise Exception('Failed to create database extensions! Exit code: %i' % exit_code)

    app = create_app()
    with app.app_context():
        db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)

        print('Creating tables...')
        db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))

        print('Creating primary and foreign keys...')
        db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
        db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))

        print('Creating indexes...')
        db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))

    print("Done!")
Example #19
0
def init_test_db(force=False):
    """Same as `init_db` command, but creates a database that will be used to
    run tests and doesn't import data (no need to do that).

    `PG_CONNECT_TEST` variable must be defined in the config file.
    """

    uri = urlsplit(create_app().config['TEST_SQLALCHEMY_DATABASE_URI'])
    if force:
        exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER +
                                    ' -h ' + uri.hostname + ' -p ' + str(uri.port) + ' < ' +
                                    os.path.join(ADMIN_SQL_DIR, 'drop_test_db.sql'),
                                    shell=True)
        if exit_code != 0:
            raise Exception('Failed to drop existing database and user! Exit code: %i' % exit_code)

    print('Creating database and user for testing...')
    exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER +
                                ' -h ' + uri.hostname + ' -p ' + str(uri.port) + ' < ' +
                                os.path.join(ADMIN_SQL_DIR, 'create_test_db.sql'),
                                shell=True)
    if exit_code != 0:
        raise Exception('Failed to create new database and user! Exit code: %i' % exit_code)

    exit_code = subprocess.call('psql -U ' + config.PG_SUPER_USER + ' -d lb_test ' +
                                ' -h ' + uri.hostname + ' -p ' + str(uri.port) + ' < ' +
                                os.path.join(ADMIN_SQL_DIR, 'create_extensions.sql'),
                                shell=True)
    if exit_code != 0:
        raise Exception('Failed to create database extensions! Exit code: %i' % exit_code)

    db.init_db_connection(config.TEST_SQLALCHEMY_DATABASE_URI)

    db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
    db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
    db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
    db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))

    print("Done!")
Example #20
0
def runserver(host, port, debug):
    app = create_app()
    schedule_jobs(app)
    app.run(host=host, port=port, debug=debug)
#!/usr/bin/env python
from webserver import create_app
import argparse
from werkzeug.contrib.profiler import ProfilerMiddleware

application = create_app()
#application.config['PROFILE'] = True
#application.wsgi_app = ProfilerMiddleware(application.wsgi_app, restrictions=[30])

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="ListenBrainz Server")
    parser.add_argument("-d", "--debug", action="store_true",
                        help="Turn on debugging mode to see stack traces on "
                             "the error pages. This overrides 'DEBUG' value "
                             "in config file.")
    parser.add_argument("-t", "--host", default="0.0.0.0", type=str,
                        help="Which interfaces to listen on. Default: 0.0.0.0.")
    parser.add_argument("-p", "--port", default="8080", type=int,
                        help="Which port to listen on. Default: 8080.")
    args = parser.parse_args()
    application.run(debug=True if args.debug else None,
                    host=args.host, port=args.port)
Example #22
0
from __future__ import print_function
import db
from webserver import create_app, schedule_jobs
from werkzeug.wsgi import DispatcherMiddleware
import subprocess
import os
import click
import config
from urlparse import urlsplit

application = DispatcherMiddleware(create_app())

cli = click.Group()

ADMIN_SQL_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'admin', 'sql')
MSB_ADMIN_SQL_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../messybrainz-server', 'admin', 'sql')

@cli.command()
@click.option("--host", "-h", default="0.0.0.0", show_default=True)
@click.option("--port", "-p", default=8080, show_default=True)
@click.option("--debug", "-d", type=bool,
              help="Turns debugging mode on or off. If specified, overrides "
                   "'DEBUG' value in the config file.")
def runserver(host, port, debug):
    app = create_app()
    schedule_jobs(app)
    app.run(host=host, port=port, debug=debug)


@cli.command()
@click.option("--force", "-f", is_flag=True, help="Drop existing database and user.")
def runserver(host, port, debug):
    create_app().run(host=host, port=port, debug=debug,
                     extra_files=config.RELOAD_ON_FILES)
Example #24
0
def runserver(host, port, debug):
    app = create_app(debug=debug)
    reload_on_files = app.config['RELOAD_ON_FILES']
    app.run(host=host, port=port, extra_files=reload_on_files)
Example #25
0
def runserver(host, port):
    create_app().run(host=host, port=port)
Example #26
0
                    broker=app.config['CELERY_BROKER_URL'])
    db.init_db_engine(app.config['SQLALCHEMY_DATABASE_URI'])
    TaskBase = celery.Task

    class ContextTask(TaskBase):
        abstract = True

        def __call__(self, *args, **kwargs):
            with app.app_context():
                return TaskBase.__call__(self, *args, **kwargs)

    celery.Task = ContextTask
    return celery


app = create_app(web=False)
celery = make_celery(app)


@celery.task()
def scrape_musicbrainz(recording_mbid):
    """

    :param recording_mbid:
    :return:
    """

    source = data.load_source("musicbrainz")
    scrapers = data.load_scrapers_for_source(source)
    if scrapers:
        s = scrapers[0]
Example #27
0
 def create_app():
     return create_app()
Example #28
0
def cache_stats():
    """Compute recent stats and add to cache."""
    # inits the db engine and cache
    create_app()
    db.stats.add_stats_to_cache()
Example #29
0
 def create_app():
     root_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                             '..')
     app = create_app(config_path=os.path.join(root_dir, 'test_config.py'))
     return app
Example #30
0
 def create_app(self):
     app = create_app()
     app.config['TESTING'] = True
     return app
Example #31
0
def runserver(host, port, debug):
    create_app().run(host=host,
                     port=port,
                     debug=debug,
                     extra_files=config.RELOAD_ON_FILES)
Example #32
0
#!/usr/bin/env python

from webserver import create_app
import argparse

application = create_app()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='AcousticBrainz server')
    parser.add_argument('-d',
                        '--debug',
                        action='store_true',
                        help='Turn on debugging mode to see stack traces on '
                        'the error pages. This overrides DEBUG value '
                        'in config file.')
    parser.add_argument('-h',
                        default='0.0.0.0',
                        type=str,
                        help='which interface to listen on. Default: 0.0.0.0')
    parser.add_argument('-p',
                        '--port',
                        default=8080,
                        type=int,
                        help='which port to listen on. Default: 8080')
    args = parser.parse_args()
    application.run(debug=True if args.debug else None,
                    host=args.host,
                    port=args.port)
 def create_app():
     app = create_app(debug=False)
     app.config['WTF_CSRF_ENABLED'] = False
     app.config['TESTING'] = True
     return app
def runserver(host, port, debug):
    db.init_db_connection(config.PG_CONNECT)
    create_app().run(host=host, port=port, debug=debug)
Example #35
0
def clear_cache():
    """Clear the cache"""
    # inits the db engine and cache
    create_app()
    cache.flush_all()
Example #36
0
from webserver import create_app

app = create_app()

app.run(debug=True)
Example #37
0
def runserver(host, port, debug):
    create_app().run(host=host, port=port, debug=debug)
Example #38
0
 def create_app(self):
     app = create_app()
     app.config['TESTING'] = True
     return app
Example #39
0
from __future__ import print_function
import db
import webserver
from werkzeug.serving import run_simple
import subprocess
import os
import click
import config
from urlparse import urlsplit


application = webserver.create_app()

cli = click.Group()

ADMIN_SQL_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'admin', 'sql')
MSB_ADMIN_SQL_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../messybrainz', 'admin', 'sql')


@cli.command()
@click.option("--host", "-h", default="0.0.0.0", show_default=True)
@click.option("--port", "-p", default=8080, show_default=True)
@click.option("--debug", "-d", is_flag=True,
              help="Turns debugging mode on or off. If specified, overrides "
                   "'DEBUG' value in the config file.")
def runserver(host, port, debug=False):
    webserver.schedule_jobs(application)
    run_simple(
        hostname=host,
        port=port,
        application=application,