def test_import_submission(conn): # first submission id = insert_submission(conn, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, 'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142', 'puid': '7c1c6753-c834-44b1-884a-a5166c093139', }) query = tables.submission.select(tables.submission.c.id == id) submission = conn.execute(query).fetchone() assert_false(submission['handled']) fingerprint = import_submission(conn, submission) assert_equals(1, fingerprint['id']) assert_equals(5, fingerprint['track_id']) query = tables.submission.select(tables.submission.c.id == id) submission = conn.execute(query).fetchone() assert_true(submission['handled']) # second submission id = insert_submission(conn, { 'fingerprint': TEST_2_FP_RAW, 'length': TEST_2_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, }) query = tables.submission.select(tables.submission.c.id == id) submission = conn.execute(query).fetchone() assert_false(submission['handled']) fingerprint = import_submission(conn, submission) assert_equals(2, fingerprint['id']) assert_equals(6, fingerprint['track_id']) query = tables.submission.select(tables.submission.c.id == id) submission = conn.execute(query).fetchone() assert_true(submission['handled']) # third submission (same as the first one) id = insert_submission(conn, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, 'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142', 'puid': '7c1c6753-c834-44b1-884a-a5166c093139', }) query = tables.submission.select(tables.submission.c.id == id) submission = conn.execute(query).fetchone() assert_false(submission['handled']) fingerprint = import_submission(conn, submission) assert_equals(1, fingerprint['id']) assert_equals(5, fingerprint['track_id']) query = tables.submission.select(tables.submission.c.id == id) submission = conn.execute(query).fetchone() assert_true(submission['handled'])
def test_import_submission_with_foreignid(ctx): # type: (ScriptContext) -> None ingest_db = ctx.db.get_ingest_db() app_db = ctx.db.get_app_db() fingerprint_db = ctx.db.get_fingerprint_db() prepare_database( fingerprint_db, """ INSERT INTO foreignid_vendor (id, name) VALUES (1, 'foo'); INSERT INTO foreignid (id, vendor_id, name) VALUES (1, 1, '123'); """) submission_id = insert_submission( ingest_db, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, 'foreignid_id': 1, }) query = tables.submission.select(tables.submission.c.id == submission_id) submission = ingest_db.execute(query).fetchone() fingerprint = import_submission(ingest_db, app_db, fingerprint_db, ctx.index, submission) assert fingerprint is not None query = tables.track_foreignid.select( tables.track_foreignid.c.track_id == fingerprint['track_id']) track_foreignid = fingerprint_db.execute(query).fetchone() assert_equals(1, track_foreignid['submission_count']) submission_id = insert_submission( ingest_db, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, 'foreignid': 'foo:123', }) query = tables.submission.select(tables.submission.c.id == submission_id) submission = ingest_db.execute(query).fetchone() fingerprint = import_submission(ingest_db, app_db, fingerprint_db, ctx.index, submission) assert fingerprint is not None query = tables.track_foreignid.select( tables.track_foreignid.c.track_id == fingerprint['track_id']) track_foreignid = fingerprint_db.execute(query).fetchone() assert_equals(2, track_foreignid['submission_count'])
def test_import_submission_with_meta(ctx): # type: (ScriptContext) -> None ingest_db = ctx.db.get_ingest_db() app_db = ctx.db.get_app_db() fingerprint_db = ctx.db.get_fingerprint_db() submission_id = insert_submission( ingest_db, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'source_id': 1, 'meta': { 'track': 'Foo' } }) query = tables.submission.select(tables.submission.c.id == submission_id) submission = ingest_db.execute(query).fetchone() fingerprint = import_submission(ingest_db, app_db, fingerprint_db, ctx.index, submission) assert fingerprint is not None query = tables.track_meta.select( tables.track_meta.c.track_id == fingerprint['track_id']) track_meta = fingerprint_db.execute(query).fetchone() assert_equals(1, track_meta['submission_count']) query = tables.meta.select(tables.meta.c.id == track_meta['meta_id']) meta = fingerprint_db.execute(query).fetchone() assert_equals('Foo', meta['track']) assert_equals(uuid.UUID('da570fc1-ecfd-5fcd-86d7-009daa0f79e5'), meta['gid'])
def test_import_submission_new_track_different(ctx): # type: (ScriptContext) -> None ingest_db = ctx.db.get_ingest_db() app_db = ctx.db.get_app_db() fingerprint_db = ctx.db.get_fingerprint_db() prepare_database( fingerprint_db, """ INSERT INTO fingerprint (fingerprint, length, track_id, submission_count) VALUES (%(fp)s, %(len)s, 1, 1); """, dict(fp=TEST_1A_FP_RAW, len=TEST_1A_LENGTH)) submission_id = insert_submission( ingest_db, { 'fingerprint': TEST_2_FP_RAW, 'length': TEST_2_LENGTH, 'source_id': 1, 'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142', 'puid': '7c1c6753-c834-44b1-884a-a5166c093139', }) query = tables.submission.select(tables.submission.c.id == submission_id) submission = ingest_db.execute(query).fetchone() assert_false(submission['handled']) fingerprint = import_submission(ingest_db, app_db, fingerprint_db, ctx.index, submission) assert fingerprint is not None assert_equals(2, fingerprint['id']) assert_equals(5, fingerprint['track_id'])
def test_import_submission_merge_existing_tracks(conn): prepare_database(conn, """ INSERT INTO fingerprint (fingerprint, length, track_id, submission_count) VALUES (%(fp1)s, %(len1)s, 1, 1), (%(fp2)s, %(len2)s, 2, 1); """, dict(fp1=TEST_1A_FP_RAW, len1=TEST_1A_LENGTH, fp2=TEST_1B_FP_RAW, len2=TEST_1B_LENGTH)) id = insert_submission(conn, { 'fingerprint': TEST_1C_FP_RAW, 'length': TEST_1C_LENGTH, 'source_id': 1, 'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142', 'puid': '7c1c6753-c834-44b1-884a-a5166c093139', }) query = tables.submission.select(tables.submission.c.id == id) submission = conn.execute(query).fetchone() assert_false(submission['handled']) try: old_threshold = const.FINGERPRINT_MERGE_THRESHOLD const.FINGERPRINT_MERGE_THRESHOLD = 0.85 fingerprint = import_submission(conn, submission) finally: const.FINGERPRINT_MERGE_THRESHOLD = old_threshold assert_equals(1, fingerprint['id']) assert_equals(1, fingerprint['track_id']) query = tables.fingerprint.select(tables.fingerprint.c.id == 1) fingerprint = conn.execute(query).fetchone() assert_equals(1, fingerprint['track_id']) query = tables.track.select(tables.track.c.id == 1) track = conn.execute(query).fetchone() assert_equals(None, track['new_id']) query = tables.track.select(tables.track.c.id == 2) track = conn.execute(query).fetchone() assert_equals(1, track['new_id'])
def test_import_submission_new_track(conn): prepare_database( conn, """ INSERT INTO fingerprint (fingerprint, length, track_id, submission_count) VALUES (%(fp)s, %(len)s, 1, 1); """, dict(fp=TEST_1A_FP_RAW, len=TEST_1A_LENGTH)) id = insert_submission( conn, { 'fingerprint': TEST_1D_FP_RAW, 'length': TEST_1D_LENGTH, 'source_id': 1, 'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142', 'puid': '7c1c6753-c834-44b1-884a-a5166c093139', }) query = tables.submission.select(tables.submission.c.id == id) submission = conn.execute(query).fetchone() assert_false(submission['handled']) try: old_threshold = const.TRACK_MERGE_THRESHOLD const.TRACK_MERGE_THRESHOLD = 0.9 fingerprint = import_submission(conn, submission) finally: const.TRACK_MERGE_THRESHOLD = old_threshold assert_equals(2, fingerprint['id']) assert_equals(5, fingerprint['track_id'])
def test_backfill_meta_gid_update(ctx): app_db = ctx.db.get_app_db() fingerprint_db = ctx.db.get_fingerprint_db() ingest_db = ctx.db.get_ingest_db() submission_id = insert_submission( ingest_db, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'source_id': 1, 'meta': { 'track': 'Foo' } }) query = tables.submission.select(tables.submission.c.id == submission_id) submission = ingest_db.execute(query).fetchone() assert submission is not None fingerprint = import_submission(ingest_db, app_db, fingerprint_db, ctx.index, submission) assert fingerprint is not None query = tables.track_meta.select( tables.track_meta.c.track_id == fingerprint['track_id']) track_meta = fingerprint_db.execute(query).fetchone() assert_equals(1, track_meta['submission_count']) query = tables.meta.update().values({ 'gid': None }).where(tables.meta.c.id == track_meta['meta_id']) fingerprint_db.execute(query) query = tables.meta.select().where( tables.meta.c.id == track_meta['meta_id']) meta = fingerprint_db.execute(query).fetchone() assert_equals(None, meta['gid']) query = tables.submission_result.update().values({ 'meta_gid': None }).where(tables.submission_result.c.meta_id == track_meta['meta_id']) ingest_db.execute(query) last_meta_id = backfill_meta_gid(fingerprint_db, ingest_db, 0, 100) assert_equals(3, last_meta_id) query = tables.meta.select().where( tables.meta.c.id == track_meta['meta_id']) meta = fingerprint_db.execute(query).fetchone() assert_equals(uuid.UUID('da570fc1-ecfd-5fcd-86d7-009daa0f79e5'), meta['gid'])
def test_import_submission_new_track_different(conn): prepare_database(conn, """ INSERT INTO fingerprint (fingerprint, length, track_id, submission_count) VALUES (%(fp)s, %(len)s, 1, 1); """, dict(fp=TEST_1A_FP_RAW, len=TEST_1A_LENGTH)) id = insert_submission(conn, { 'fingerprint': TEST_2_FP_RAW, 'length': TEST_2_LENGTH, 'source_id': 1, 'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142', 'puid': '7c1c6753-c834-44b1-884a-a5166c093139', }) query = tables.submission.select(tables.submission.c.id == id) submission = conn.execute(query).fetchone() assert_false(submission['handled']) fingerprint = import_submission(conn, submission) assert_equals(2, fingerprint['id']) assert_equals(5, fingerprint['track_id'])
def test_import_submission_reuse_fingerprint_100(conn): prepare_database(conn, """ INSERT INTO fingerprint (fingerprint, length, track_id, submission_count) VALUES (%(fp)s, %(len)s, 1, 1); """, dict(fp=TEST_1A_FP_RAW, len=TEST_1A_LENGTH)) id = insert_submission(conn, { 'fingerprint': TEST_1A_FP_RAW, 'length': TEST_1A_LENGTH, 'source_id': 1, 'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142', 'puid': '7c1c6753-c834-44b1-884a-a5166c093139', }) query = tables.submission.select(tables.submission.c.id == id) submission = conn.execute(query).fetchone() assert_false(submission['handled']) fingerprint = import_submission(conn, submission) assert_equals(1, fingerprint['id']) assert_equals(1, fingerprint['track_id'])
def test_import_submission_with_foreignid(conn): prepare_database(conn, """ INSERT INTO foreignid_vendor (id, name) VALUES (1, 'foo'); INSERT INTO foreignid (id, vendor_id, name) VALUES (1, 1, '123'); """) id = insert_submission(conn, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, 'foreignid_id': 1, }) query = tables.submission.select(tables.submission.c.id == id) submission = conn.execute(query).fetchone() fingerprint = import_submission(conn, submission) query = tables.track_foreignid.select(tables.track_foreignid.c.track_id == fingerprint['track_id']) track_foreignid = conn.execute(query).fetchone() assert_equals(1, track_foreignid['submission_count'])
def test_import_submission(ctx): # type: (ScriptContext) -> None ingest_db = ctx.db.get_ingest_db() app_db = ctx.db.get_app_db() fingerprint_db = ctx.db.get_fingerprint_db() # first submission submission_id = insert_submission( ingest_db, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'account_id': 1, 'application_id': 1, 'format': 'FLAC', 'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142', 'puid': '7c1c6753-c834-44b1-884a-a5166c093139', }) query = tables.submission.select(tables.submission.c.id == submission_id) submission = ingest_db.execute(query).fetchone() assert_false(submission['handled']) fingerprint = import_submission(ingest_db, app_db, fingerprint_db, ctx.index, submission) assert fingerprint is not None assert_equals(1, fingerprint['id']) assert_equals(5, fingerprint['track_id']) query = tables.submission.select(tables.submission.c.id == submission_id) submission = ingest_db.execute(query).fetchone() assert_true(submission['handled']) query = tables.track_mbid.select( tables.track_mbid.c.track_id == fingerprint['track_id']) track_mbid = fingerprint_db.execute(query).fetchone() assert_equals(1, track_mbid['submission_count']) query = tables.track_puid.select( tables.track_puid.c.track_id == fingerprint['track_id']) track_puid = fingerprint_db.execute(query).fetchone() assert_equals(1, track_puid['submission_count']) query = tables.fingerprint.select( tables.fingerprint.c.id == fingerprint['id']) fingerprint = fingerprint_db.execute(query).fetchone() assert fingerprint is not None assert_equals(1, fingerprint['submission_count']) assert_equals(1, fingerprint['format_id']) # second submission submission_id = insert_submission( ingest_db, { 'fingerprint': TEST_2_FP_RAW, 'length': TEST_2_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, }) query = tables.submission.select(tables.submission.c.id == submission_id) submission = ingest_db.execute(query).fetchone() assert_false(submission['handled']) fingerprint = import_submission(ingest_db, app_db, fingerprint_db, ctx.index, submission) assert fingerprint is not None assert_equals(2, fingerprint['id']) assert_equals(6, fingerprint['track_id']) query = tables.submission.select(tables.submission.c.id == submission_id) submission = ingest_db.execute(query).fetchone() assert_true(submission['handled']) # third submission (same as the first one) submission_id = insert_submission( ingest_db, { 'fingerprint': TEST_1_FP_RAW, 'length': TEST_1_LENGTH, 'bitrate': 192, 'source_id': 1, 'format_id': 1, 'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142', 'puid': '7c1c6753-c834-44b1-884a-a5166c093139', }) query = tables.submission.select(tables.submission.c.id == submission_id) submission = ingest_db.execute(query).fetchone() assert_false(submission['handled']) fingerprint = import_submission(ingest_db, app_db, fingerprint_db, ctx.index, submission) assert fingerprint is not None assert_equals(1, fingerprint['id']) assert_equals(5, fingerprint['track_id']) query = tables.submission.select(tables.submission.c.id == submission_id) submission = ingest_db.execute(query).fetchone() assert_true(submission['handled']) query = tables.track_mbid.select( tables.track_mbid.c.track_id == fingerprint['track_id']) track_mbid = fingerprint_db.execute(query).fetchone() assert_equals(2, track_mbid['submission_count']) query = tables.track_puid.select( tables.track_puid.c.track_id == fingerprint['track_id']) track_puid = fingerprint_db.execute(query).fetchone() assert_equals(2, track_puid['submission_count']) query = tables.fingerprint.select( tables.fingerprint.c.id == fingerprint['id']) fingerprint = fingerprint_db.execute(query).fetchone() assert fingerprint is not None assert_equals(2, fingerprint['submission_count'])