def lstokens():
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
tokens = data.get_tokens()
for t in tokens:
print("%s admin=%s" % (t["token"], t["admin"]))
def init_test_db(force=False):
"""Same as `init_db` command, but creates a database that will be used to run tests.
`TEST_SQLALCHEMY_DATABASE_URI` variable must be defined in the config file.
"""
if force:
exit_code = _run_psql('drop_test_db.sql')
if exit_code != 0:
raise Exception('Failed to drop existing database and user! Exit code: %i' % exit_code)
print('Creating database and user for testing...')
exit_code = _run_psql('create_test_db.sql')
if exit_code != 0:
raise Exception('Failed to create new database and user! Exit code: %i' % exit_code)
exit_code = _run_psql('create_extensions.sql', 'mdb_test')
if exit_code != 0:
raise Exception('Failed to create database extensions! Exit code: %i' % exit_code)
db.init_db_engine(config.TEST_SQLALCHEMY_DATABASE_URI)
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def init_test_db(force=False):
"""Same as `init_db` command, but creates a database that will be used to run tests.
`SQLALCHEMY_TEST_URI` variable must be defined in the config file.
"""
db.init_db_engine(config.SQLALCHEMY_ADMIN_URI)
if force:
print('Dropping existing test database...')
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'drop_test_db.sql'))
if not res:
sys.exit(1)
print('Creating database and user for testing...')
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'create_test_db.sql'))
if not res:
sys.exit(1)
db.engine.dispose()
db.init_db_engine(config.SQLALCHEMY_TEST_URI)
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_schema.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def main():
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
source = data.load_source("musicbrainz")
scraper = data.load_latest_scraper_for_source(source)
print(scraper)
mbids = get_mbids()
total = len(mbids)
starttime = time.time()
done = 0
for mblist in util.chunks(mbids, 100):
lookup(mblist, scraper)
done += 100
durdelta, remdelta = util.stats(done, total, starttime)
log.info("Done %s/%s in %s; %s remaining", done, total, str(durdelta), str(remdelta))
def make_celery(app):
celery = Celery(app.import_name,
backend=app.config['CELERY_RESULT_BACKEND'],
broker=app.config['CELERY_BROKER_URL'])
db.init_db_engine(app.config['SQLALCHEMY_DATABASE_URI'])
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
def main():
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
source = data.load_source("musicbrainz")
scraper = data.load_latest_scraper_for_source(source)
print(scraper)
mbids = get_mbids()
total = len(mbids)
starttime = time.time()
done = 0
for mblist in util.chunks(mbids, 100):
lookup(mblist, scraper)
done += 100
durdelta, remdelta = util.stats(done, total, starttime)
log.info("Done %s/%s in %s; %s remaining", done, total, str(durdelta),
str(remdelta))
def fixtures():
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
sources = os.path.join("fixtures", "sources")
with open(sources) as fp:
r = csv.DictReader(fp)
for line in r:
if not data.load_source(**line):
data.add_source(**line)
scrapers = os.path.join("fixtures", "scrapers")
with open(scrapers) as fp:
r = csv.DictReader(fp)
for line in r:
sname = line.pop("source")
source = data.load_source(sname)
line["source"] = source
data.add_scraper(**line)
def fixtures():
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
sources = os.path.join("fixtures", "sources")
with open(sources) as fp:
r = csv.DictReader(fp)
for line in r:
if not data.load_source(**line):
data.add_source(**line)
scrapers = os.path.join("fixtures", "scrapers")
with open(scrapers) as fp:
r = csv.DictReader(fp)
for line in r:
sname = line.pop("source")
source = data.load_source(sname)
line["source"] = source
data.add_scraper(**line)
def main():
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
source = data.load_source("musicbrainz")
scraper = data.load_latest_scraper_for_source(source)
recordings = get_recordings()
total = len(recordings)
done = 0
starttime = time.time()
log.info("starting..., %s recordings to process", total)
for reclist in util.chunks(recordings, 10000):
log.info("have %s recordings", len(reclist))
with db.engine.connect() as connection:
saveddata = get_data(connection, scraper["id"], reclist)
log.info(" - got %s rows matching them", len(saveddata))
process(connection, saveddata)
done += len(reclist)
durdelta, remdelta = util.stats(done, total, starttime)
log.info("Done %s/%s in %s; %s remaining", done, total, str(durdelta), str(remdelta))
def init_db(force):
"""Initializes database.
This process involves several steps:
1. Table structure is created.
2. Primary keys and foreign keys are created.
3. Indexes are created.
"""
if force:
exit_code = _run_psql('drop_db.sql')
if exit_code != 0:
raise Exception(
'Failed to drop existing database and user! Exit code: %i' %
exit_code)
print('Creating user and a database...')
exit_code = _run_psql('create_db.sql')
if exit_code != 0:
raise Exception(
'Failed to create new database and user! Exit code: %i' %
exit_code)
print('Creating database extensions...')
exit_code = _run_psql('create_extensions.sql', 'metadb')
if exit_code != 0:
raise Exception('Failed to create database extensions! Exit code: %i' %
exit_code)
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
print('Creating tables...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
print('Creating primary and foreign keys...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
print('Creating indexes...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def main():
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
source = data.load_source("musicbrainz")
scraper = data.load_latest_scraper_for_source(source)
recordings = get_recordings()
total = len(recordings)
done = 0
starttime = time.time()
log.info("starting..., %s recordings to process", total)
for reclist in util.chunks(recordings, 10000):
log.info("have %s recordings", len(reclist))
with db.engine.connect() as connection:
saveddata = get_data(connection, scraper["id"], reclist)
log.info(" - got %s rows matching them", len(saveddata))
process(connection, saveddata)
done += len(reclist)
durdelta, remdelta = util.stats(done, total, starttime)
log.info("Done %s/%s in %s; %s remaining", done, total, str(durdelta),
str(remdelta))
def main():
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
log.info("Release groups")
releasegroups = get_rgs()
fieldnames = ["mbid", "release_title", "artist", "year"]
with open("release-group-meta.csv", "w") as fp:
w = csv.DictWriter(fp, fieldnames=fieldnames)
w.writeheader()
for rg in releasegroups:
w.writerow(rg)
log.info("Recordings")
recordings = get_recordings()
fieldnames = ["mbid", "recording", "artist"]
count = (len(recordings) // 8) + 1
for i, reclist in enumerate(util.chunks(recordings, count), 1):
with open("recording-meta-{}.csv".format(i), "w") as fp:
w = csv.DictWriter(fp, fieldnames=fieldnames)
w.writeheader()
for rec in reclist:
w.writerow(rec)
def main():
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
log.info("Release groups")
releasegroups = get_rgs()
fieldnames = ["mbid", "release_title", "artist", "year"]
with open("release-group-meta.csv", "w") as fp:
w = csv.DictWriter(fp, fieldnames=fieldnames)
w.writeheader()
for rg in releasegroups:
w.writerow(rg)
log.info("Recordings")
recordings = get_recordings()
fieldnames = ["mbid", "recording", "artist"]
count = (len(recordings)//8) + 1
for i, reclist in enumerate(util.chunks(recordings, count), 1):
with open("recording-meta-{}.csv".format(i), "w") as fp:
w = csv.DictWriter(fp, fieldnames=fieldnames)
w.writeheader()
for rec in reclist:
w.writerow(rec)
def init_db(force):
"""Initializes database.
This process involves several steps:
1. Table structure is created.
2. Primary keys and foreign keys are created.
3. Indexes are created.
"""
db.init_db_engine(config.SQLALCHEMY_ADMIN_URI)
if force:
print('Dropping existing database...')
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'drop_db.sql'))
if not res:
sys.exit(1)
print('Creating user and a database...')
res = db.run_sql_script_without_transaction(
os.path.join(ADMIN_SQL_DIR, 'create_db.sql'))
if not res:
sys.exit(1)
db.engine.dispose()
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
print('Creating tables...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_schema.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
print('Creating primary and foreign keys...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
print('Creating indexes...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def init_db(force):
"""Initializes database.
This process involves several steps:
1. Table structure is created.
2. Primary keys and foreign keys are created.
3. Indexes are created.
"""
if force:
exit_code = _run_psql('drop_db.sql')
if exit_code != 0:
raise Exception('Failed to drop existing database and user! Exit code: %i' % exit_code)
print('Creating user and a database...')
exit_code = _run_psql('create_db.sql')
if exit_code != 0:
raise Exception('Failed to create new database and user! Exit code: %i' % exit_code)
print('Creating database extensions...')
exit_code = _run_psql('create_extensions.sql', 'metadb')
if exit_code != 0:
raise Exception('Failed to create database extensions! Exit code: %i' % exit_code)
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
print('Creating tables...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_tables.sql'))
print('Creating primary and foreign keys...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_primary_keys.sql'))
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_foreign_keys.sql'))
print('Creating indexes...')
db.run_sql_script(os.path.join(ADMIN_SQL_DIR, 'create_indexes.sql'))
print("Done!")
def addtoken(admin):
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
token = data.add_token(admin)
print("Added token: %s" % token)
def before_request():
db.init_db_engine(app.config['SQLALCHEMY_DATABASE_URI'])
def before_request():
db.init_db_engine(app.config['SQLALCHEMY_DATABASE_URI'])
def setUp(self):
db.init_db_engine(config.SQLALCHEMY_TEST_URI)
self.reset_db()
def rmtoken(token):
db.init_db_engine(config.SQLALCHEMY_DATABASE_URI)
data.remove_token(token)
