def test_import_submission(conn):
# first submission
id = insert_submission(conn, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(conn, submission)
assert_equals(1, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_true(submission['handled'])
# second submission
id = insert_submission(conn, {
'fingerprint': TEST_2_FP_RAW,
'length': TEST_2_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(conn, submission)
assert_equals(2, fingerprint['id'])
assert_equals(6, fingerprint['track_id'])
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_true(submission['handled'])
# third submission (same as the first one)
id = insert_submission(conn, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(conn, submission)
assert_equals(1, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_true(submission['handled'])
def test_import_submission_with_foreignid(ctx):
# type: (ScriptContext) -> None
ingest_db = ctx.db.get_ingest_db()
app_db = ctx.db.get_app_db()
fingerprint_db = ctx.db.get_fingerprint_db()
prepare_database(
fingerprint_db, """
INSERT INTO foreignid_vendor (id, name) VALUES (1, 'foo');
INSERT INTO foreignid (id, vendor_id, name) VALUES (1, 1, '123');
""")
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'foreignid_id': 1,
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
query = tables.track_foreignid.select(
tables.track_foreignid.c.track_id == fingerprint['track_id'])
track_foreignid = fingerprint_db.execute(query).fetchone()
assert_equals(1, track_foreignid['submission_count'])
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'foreignid': 'foo:123',
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
query = tables.track_foreignid.select(
tables.track_foreignid.c.track_id == fingerprint['track_id'])
track_foreignid = fingerprint_db.execute(query).fetchone()
assert_equals(2, track_foreignid['submission_count'])
def test_import_submission_with_meta(ctx):
# type: (ScriptContext) -> None
ingest_db = ctx.db.get_ingest_db()
app_db = ctx.db.get_app_db()
fingerprint_db = ctx.db.get_fingerprint_db()
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'meta': {
'track': 'Foo'
}
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
query = tables.track_meta.select(
tables.track_meta.c.track_id == fingerprint['track_id'])
track_meta = fingerprint_db.execute(query).fetchone()
assert_equals(1, track_meta['submission_count'])
query = tables.meta.select(tables.meta.c.id == track_meta['meta_id'])
meta = fingerprint_db.execute(query).fetchone()
assert_equals('Foo', meta['track'])
assert_equals(uuid.UUID('da570fc1-ecfd-5fcd-86d7-009daa0f79e5'),
meta['gid'])
def test_import_submission_new_track_different(ctx):
# type: (ScriptContext) -> None
ingest_db = ctx.db.get_ingest_db()
app_db = ctx.db.get_app_db()
fingerprint_db = ctx.db.get_fingerprint_db()
prepare_database(
fingerprint_db, """
INSERT INTO fingerprint (fingerprint, length, track_id, submission_count)
VALUES (%(fp)s, %(len)s, 1, 1);
""", dict(fp=TEST_1A_FP_RAW, len=TEST_1A_LENGTH))
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_2_FP_RAW,
'length': TEST_2_LENGTH,
'source_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
assert_equals(2, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
def test_import_submission_merge_existing_tracks(conn):
prepare_database(conn, """
INSERT INTO fingerprint (fingerprint, length, track_id, submission_count)
VALUES (%(fp1)s, %(len1)s, 1, 1), (%(fp2)s, %(len2)s, 2, 1);
""", dict(fp1=TEST_1A_FP_RAW, len1=TEST_1A_LENGTH,
fp2=TEST_1B_FP_RAW, len2=TEST_1B_LENGTH))
id = insert_submission(conn, {
'fingerprint': TEST_1C_FP_RAW,
'length': TEST_1C_LENGTH,
'source_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
try:
old_threshold = const.FINGERPRINT_MERGE_THRESHOLD
const.FINGERPRINT_MERGE_THRESHOLD = 0.85
fingerprint = import_submission(conn, submission)
finally:
const.FINGERPRINT_MERGE_THRESHOLD = old_threshold
assert_equals(1, fingerprint['id'])
assert_equals(1, fingerprint['track_id'])
query = tables.fingerprint.select(tables.fingerprint.c.id == 1)
fingerprint = conn.execute(query).fetchone()
assert_equals(1, fingerprint['track_id'])
query = tables.track.select(tables.track.c.id == 1)
track = conn.execute(query).fetchone()
assert_equals(None, track['new_id'])
query = tables.track.select(tables.track.c.id == 2)
track = conn.execute(query).fetchone()
assert_equals(1, track['new_id'])
def test_import_submission_new_track(conn):
prepare_database(
conn, """
INSERT INTO fingerprint (fingerprint, length, track_id, submission_count)
VALUES (%(fp)s, %(len)s, 1, 1);
""", dict(fp=TEST_1A_FP_RAW, len=TEST_1A_LENGTH))
id = insert_submission(
conn, {
'fingerprint': TEST_1D_FP_RAW,
'length': TEST_1D_LENGTH,
'source_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
try:
old_threshold = const.TRACK_MERGE_THRESHOLD
const.TRACK_MERGE_THRESHOLD = 0.9
fingerprint = import_submission(conn, submission)
finally:
const.TRACK_MERGE_THRESHOLD = old_threshold
assert_equals(2, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
def test_backfill_meta_gid_update(ctx):
app_db = ctx.db.get_app_db()
fingerprint_db = ctx.db.get_fingerprint_db()
ingest_db = ctx.db.get_ingest_db()
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'meta': {
'track': 'Foo'
}
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert submission is not None
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
query = tables.track_meta.select(
tables.track_meta.c.track_id == fingerprint['track_id'])
track_meta = fingerprint_db.execute(query).fetchone()
assert_equals(1, track_meta['submission_count'])
query = tables.meta.update().values({
'gid': None
}).where(tables.meta.c.id == track_meta['meta_id'])
fingerprint_db.execute(query)
query = tables.meta.select().where(
tables.meta.c.id == track_meta['meta_id'])
meta = fingerprint_db.execute(query).fetchone()
assert_equals(None, meta['gid'])
query = tables.submission_result.update().values({
'meta_gid': None
}).where(tables.submission_result.c.meta_id == track_meta['meta_id'])
ingest_db.execute(query)
last_meta_id = backfill_meta_gid(fingerprint_db, ingest_db, 0, 100)
assert_equals(3, last_meta_id)
query = tables.meta.select().where(
tables.meta.c.id == track_meta['meta_id'])
meta = fingerprint_db.execute(query).fetchone()
assert_equals(uuid.UUID('da570fc1-ecfd-5fcd-86d7-009daa0f79e5'),
meta['gid'])
def test_import_submission_new_track_different(conn):
prepare_database(conn, """
INSERT INTO fingerprint (fingerprint, length, track_id, submission_count)
VALUES (%(fp)s, %(len)s, 1, 1);
""", dict(fp=TEST_1A_FP_RAW, len=TEST_1A_LENGTH))
id = insert_submission(conn, {
'fingerprint': TEST_2_FP_RAW,
'length': TEST_2_LENGTH,
'source_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(conn, submission)
assert_equals(2, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
def test_import_submission_reuse_fingerprint_100(conn):
prepare_database(conn, """
INSERT INTO fingerprint (fingerprint, length, track_id, submission_count)
VALUES (%(fp)s, %(len)s, 1, 1);
""", dict(fp=TEST_1A_FP_RAW, len=TEST_1A_LENGTH))
id = insert_submission(conn, {
'fingerprint': TEST_1A_FP_RAW,
'length': TEST_1A_LENGTH,
'source_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(conn, submission)
assert_equals(1, fingerprint['id'])
assert_equals(1, fingerprint['track_id'])
def test_import_submission_with_foreignid(conn):
prepare_database(conn, """
INSERT INTO foreignid_vendor (id, name) VALUES (1, 'foo');
INSERT INTO foreignid (id, vendor_id, name) VALUES (1, 1, '123');
""")
id = insert_submission(conn, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'foreignid_id': 1,
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
fingerprint = import_submission(conn, submission)
query = tables.track_foreignid.select(tables.track_foreignid.c.track_id == fingerprint['track_id'])
track_foreignid = conn.execute(query).fetchone()
assert_equals(1, track_foreignid['submission_count'])
def test_import_submission(ctx):
# type: (ScriptContext) -> None
ingest_db = ctx.db.get_ingest_db()
app_db = ctx.db.get_app_db()
fingerprint_db = ctx.db.get_fingerprint_db()
# first submission
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'account_id': 1,
'application_id': 1,
'format': 'FLAC',
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
assert_equals(1, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_true(submission['handled'])
query = tables.track_mbid.select(
tables.track_mbid.c.track_id == fingerprint['track_id'])
track_mbid = fingerprint_db.execute(query).fetchone()
assert_equals(1, track_mbid['submission_count'])
query = tables.track_puid.select(
tables.track_puid.c.track_id == fingerprint['track_id'])
track_puid = fingerprint_db.execute(query).fetchone()
assert_equals(1, track_puid['submission_count'])
query = tables.fingerprint.select(
tables.fingerprint.c.id == fingerprint['id'])
fingerprint = fingerprint_db.execute(query).fetchone()
assert fingerprint is not None
assert_equals(1, fingerprint['submission_count'])
assert_equals(1, fingerprint['format_id'])
# second submission
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_2_FP_RAW,
'length': TEST_2_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
assert_equals(2, fingerprint['id'])
assert_equals(6, fingerprint['track_id'])
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_true(submission['handled'])
# third submission (same as the first one)
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
assert_equals(1, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_true(submission['handled'])
query = tables.track_mbid.select(
tables.track_mbid.c.track_id == fingerprint['track_id'])
track_mbid = fingerprint_db.execute(query).fetchone()
assert_equals(2, track_mbid['submission_count'])
query = tables.track_puid.select(
tables.track_puid.c.track_id == fingerprint['track_id'])
track_puid = fingerprint_db.execute(query).fetchone()
assert_equals(2, track_puid['submission_count'])
query = tables.fingerprint.select(
tables.fingerprint.c.id == fingerprint['id'])
fingerprint = fingerprint_db.execute(query).fetchone()
assert fingerprint is not None
assert_equals(2, fingerprint['submission_count'])
