def test_merge_mbids(conn):
insert_submission(conn, {'fingerprint': [1], 'length': 123, 'source_id': 1})
insert_submission(conn, {'fingerprint': [1], 'length': 123, 'source_id': 1})
prepare_database(conn, """
TRUNCATE track_mbid CASCADE;
INSERT INTO track_mbid (id, track_id, mbid, submission_count) VALUES (1, 1, '97edb73c-4dac-11e0-9096-0025225356f3', 9);
INSERT INTO track_mbid (id, track_id, mbid, submission_count) VALUES (2, 1, 'd575d506-4da4-11e0-b951-0025225356f3', 11);
INSERT INTO track_mbid_source (track_mbid_id, submission_id, source_id) VALUES (1, 1, 1);
INSERT INTO track_mbid_source (track_mbid_id, submission_id, source_id) VALUES (2, 2, 1);
INSERT INTO track_mbid_change (track_mbid_id, account_id, disabled) VALUES (1, 1, true);
INSERT INTO track_mbid_change (track_mbid_id, account_id, disabled) VALUES (2, 1, true);
""")
merge_mbids(conn, '97edb73c-4dac-11e0-9096-0025225356f3', ['d575d506-4da4-11e0-b951-0025225356f3'])
rows = conn.execute("SELECT track_id, mbid, submission_count, disabled FROM track_mbid ORDER BY track_id, mbid").fetchall()
expected_rows = [
(1, '97edb73c-4dac-11e0-9096-0025225356f3', 20, False),
]
assert_equals(expected_rows, rows)
rows = conn.execute("SELECT track_mbid_id, submission_id, source_id FROM track_mbid_source ORDER BY track_mbid_id, submission_id, source_id").fetchall()
expected_rows = [
(1, 1, 1),
(1, 2, 1),
]
assert_equals(expected_rows, rows)
rows = conn.execute("SELECT track_mbid_id, account_id FROM track_mbid_change ORDER BY track_mbid_id, account_id").fetchall()
expected_rows = [
(1, 1),
(1, 1),
]
assert_equals(expected_rows, rows)
def _handle_internal(self, params):
with self.conn.begin():
source_id = find_or_insert_source(self.conn, params.application_id, params.account_id, params.application_version)
format_ids = {}
for p in params.submissions:
if p['format']:
if p['format'] not in format_ids:
format_ids[p['format']] = find_or_insert_format(self.conn, p['format'])
p['format_id'] = format_ids[p['format']]
for p in params.submissions:
mbids = p['mbids'] or [None]
for mbid in mbids:
values = {
'mbid': mbid or None,
'puid': p['puid'] or None,
'bitrate': p['bitrate'] or None,
'fingerprint': p['fingerprint'],
'length': p['duration'],
'format_id': p.get('format_id'),
'source_id': source_id
}
meta_values = dict((n, p[n] or None) for n in self.meta_fields)
if any(meta_values.itervalues()):
values['meta_id'] = insert_meta(self.conn, meta_values)
if p['foreignid']:
values['foreignid_id'] = find_or_insert_foreignid(self.conn, p['foreignid'])
insert_submission(self.conn, values)
return {}
def _handle_internal(self, params):
with self.conn.begin():
source_id = find_or_insert_source(
self.conn, params.application_id, params.account_id, params.application_version
)
format_ids = {}
for p in params.submissions:
if p["format"]:
if p["format"] not in format_ids:
format_ids[p["format"]] = find_or_insert_format(self.conn, p["format"])
p["format_id"] = format_ids[p["format"]]
for p in params.submissions:
mbids = p["mbids"] or [None]
for mbid in mbids:
values = {
"mbid": mbid or None,
"puid": p["puid"] or None,
"bitrate": p["bitrate"] or None,
"fingerprint": p["fingerprint"],
"length": p["duration"],
"format_id": p.get("format_id"),
"source_id": source_id,
}
meta_values = dict((n, p[n] or None) for n in self.meta_fields)
if any(meta_values.itervalues()):
values["meta_id"] = insert_meta(self.conn, meta_values)
if p["foreignid"]:
values["foreignid_id"] = find_or_insert_foreignid(self.conn, p["foreignid"])
insert_submission(self.conn, values)
return {}
def test_import_submission(conn):
# first submission
id = insert_submission(conn, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(conn, submission)
assert_equals(1, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_true(submission['handled'])
# second submission
id = insert_submission(conn, {
'fingerprint': TEST_2_FP_RAW,
'length': TEST_2_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(conn, submission)
assert_equals(2, fingerprint['id'])
assert_equals(6, fingerprint['track_id'])
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_true(submission['handled'])
# third submission (same as the first one)
id = insert_submission(conn, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(conn, submission)
assert_equals(1, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_true(submission['handled'])
def test_import_submission_with_foreignid(ctx):
# type: (ScriptContext) -> None
ingest_db = ctx.db.get_ingest_db()
app_db = ctx.db.get_app_db()
fingerprint_db = ctx.db.get_fingerprint_db()
prepare_database(
fingerprint_db, """
INSERT INTO foreignid_vendor (id, name) VALUES (1, 'foo');
INSERT INTO foreignid (id, vendor_id, name) VALUES (1, 1, '123');
""")
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'foreignid_id': 1,
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
query = tables.track_foreignid.select(
tables.track_foreignid.c.track_id == fingerprint['track_id'])
track_foreignid = fingerprint_db.execute(query).fetchone()
assert_equals(1, track_foreignid['submission_count'])
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'foreignid': 'foo:123',
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
query = tables.track_foreignid.select(
tables.track_foreignid.c.track_id == fingerprint['track_id'])
track_foreignid = fingerprint_db.execute(query).fetchone()
assert_equals(2, track_foreignid['submission_count'])
def test_import_submission_new_track_different(ctx):
# type: (ScriptContext) -> None
ingest_db = ctx.db.get_ingest_db()
app_db = ctx.db.get_app_db()
fingerprint_db = ctx.db.get_fingerprint_db()
prepare_database(
fingerprint_db, """
INSERT INTO fingerprint (fingerprint, length, track_id, submission_count)
VALUES (%(fp)s, %(len)s, 1, 1);
""", dict(fp=TEST_1A_FP_RAW, len=TEST_1A_LENGTH))
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_2_FP_RAW,
'length': TEST_2_LENGTH,
'source_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
assert_equals(2, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
def test_import_submission_merge_existing_tracks(conn):
prepare_database(conn, """
INSERT INTO fingerprint (fingerprint, length, track_id, submission_count)
VALUES (%(fp1)s, %(len1)s, 1, 1), (%(fp2)s, %(len2)s, 2, 1);
""", dict(fp1=TEST_1A_FP_RAW, len1=TEST_1A_LENGTH,
fp2=TEST_1B_FP_RAW, len2=TEST_1B_LENGTH))
id = insert_submission(conn, {
'fingerprint': TEST_1C_FP_RAW,
'length': TEST_1C_LENGTH,
'source_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
try:
old_threshold = const.FINGERPRINT_MERGE_THRESHOLD
const.FINGERPRINT_MERGE_THRESHOLD = 0.85
fingerprint = import_submission(conn, submission)
finally:
const.FINGERPRINT_MERGE_THRESHOLD = old_threshold
assert_equals(1, fingerprint['id'])
assert_equals(1, fingerprint['track_id'])
query = tables.fingerprint.select(tables.fingerprint.c.id == 1)
fingerprint = conn.execute(query).fetchone()
assert_equals(1, fingerprint['track_id'])
query = tables.track.select(tables.track.c.id == 1)
track = conn.execute(query).fetchone()
assert_equals(None, track['new_id'])
query = tables.track.select(tables.track.c.id == 2)
track = conn.execute(query).fetchone()
assert_equals(1, track['new_id'])
def test_import_submission_with_meta(ctx):
# type: (ScriptContext) -> None
ingest_db = ctx.db.get_ingest_db()
app_db = ctx.db.get_app_db()
fingerprint_db = ctx.db.get_fingerprint_db()
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'meta': {
'track': 'Foo'
}
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
query = tables.track_meta.select(
tables.track_meta.c.track_id == fingerprint['track_id'])
track_meta = fingerprint_db.execute(query).fetchone()
assert_equals(1, track_meta['submission_count'])
query = tables.meta.select(tables.meta.c.id == track_meta['meta_id'])
meta = fingerprint_db.execute(query).fetchone()
assert_equals('Foo', meta['track'])
assert_equals(uuid.UUID('da570fc1-ecfd-5fcd-86d7-009daa0f79e5'),
meta['gid'])
def test_import_submission_new_track(conn):
prepare_database(
conn, """
INSERT INTO fingerprint (fingerprint, length, track_id, submission_count)
VALUES (%(fp)s, %(len)s, 1, 1);
""", dict(fp=TEST_1A_FP_RAW, len=TEST_1A_LENGTH))
id = insert_submission(
conn, {
'fingerprint': TEST_1D_FP_RAW,
'length': TEST_1D_LENGTH,
'source_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
try:
old_threshold = const.TRACK_MERGE_THRESHOLD
const.TRACK_MERGE_THRESHOLD = 0.9
fingerprint = import_submission(conn, submission)
finally:
const.TRACK_MERGE_THRESHOLD = old_threshold
assert_equals(2, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
def test_merge_mbids(conn):
insert_submission(conn, {
'fingerprint': [1],
'length': 123,
'source_id': 1
})
insert_submission(conn, {
'fingerprint': [1],
'length': 123,
'source_id': 1
})
prepare_database(
conn, """
TRUNCATE track_mbid CASCADE;
INSERT INTO track_mbid (id, track_id, mbid, submission_count) VALUES (1, 1, '97edb73c-4dac-11e0-9096-0025225356f3', 9);
INSERT INTO track_mbid (id, track_id, mbid, submission_count) VALUES (2, 1, 'd575d506-4da4-11e0-b951-0025225356f3', 11);
INSERT INTO track_mbid_source (track_mbid_id, submission_id, source_id) VALUES (1, 1, 1);
INSERT INTO track_mbid_source (track_mbid_id, submission_id, source_id) VALUES (2, 2, 1);
INSERT INTO track_mbid_change (track_mbid_id, account_id, disabled) VALUES (1, 1, true);
INSERT INTO track_mbid_change (track_mbid_id, account_id, disabled) VALUES (2, 1, true);
""")
merge_mbids(conn, '97edb73c-4dac-11e0-9096-0025225356f3',
['d575d506-4da4-11e0-b951-0025225356f3'])
rows = conn.execute(
"SELECT track_id, mbid, submission_count, disabled FROM track_mbid ORDER BY track_id, mbid"
).fetchall()
expected_rows = [
(1, UUID('97edb73c-4dac-11e0-9096-0025225356f3'), 20, False),
]
assert_equals(expected_rows, rows)
rows = conn.execute(
"SELECT track_mbid_id, submission_id, source_id FROM track_mbid_source ORDER BY track_mbid_id, submission_id, source_id"
).fetchall()
expected_rows = [
(1, 1, 1),
(1, 2, 1),
]
assert_equals(expected_rows, rows)
rows = conn.execute(
"SELECT track_mbid_id, account_id FROM track_mbid_change ORDER BY track_mbid_id, account_id"
).fetchall()
expected_rows = [
(1, 1),
(1, 1),
]
assert_equals(expected_rows, rows)
def _handle_internal(self, params):
with self.conn.begin():
source_id = find_or_insert_source(self.conn, params.application_id, params.account_id)
format_ids = {}
for p in params.submissions:
if p['format'] and p['format'] not in format_ids:
format_ids[p['format']] = find_or_insert_format(self.conn, p['format'])
for p in params.submissions:
for mbid in p['mbids']:
insert_submission(self.conn, {
'mbid': mbid or None,
'puid': p['puid'] or None,
'bitrate': p['bitrate'] or None,
'fingerprint': p['fingerprint'],
'length': p['duration'],
'format_id': format_ids[p['format']] if p['format'] else None,
'source_id': source_id
})
return {}
def test_import_queued_submissions(conn):
insert_meta(conn, {'track': 'Foo'})
insert_submission(conn, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'meta_id': 1,
})
insert_submission(conn, {
'fingerprint': TEST_2_FP_RAW,
'length': TEST_2_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
})
insert_submission(conn, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
})
import_queued_submissions(conn)
count = conn.execute("SELECT count(*) FROM fingerprint WHERE id IN (1,2,3)").scalar()
assert_equals(2, count)
count = conn.execute("SELECT count(*) FROM track WHERE id IN (5,6,7)").scalar()
assert_equals(2, count)
def test_backfill_meta_gid_update(ctx):
app_db = ctx.db.get_app_db()
fingerprint_db = ctx.db.get_fingerprint_db()
ingest_db = ctx.db.get_ingest_db()
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'meta': {
'track': 'Foo'
}
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert submission is not None
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
query = tables.track_meta.select(
tables.track_meta.c.track_id == fingerprint['track_id'])
track_meta = fingerprint_db.execute(query).fetchone()
assert_equals(1, track_meta['submission_count'])
query = tables.meta.update().values({
'gid': None
}).where(tables.meta.c.id == track_meta['meta_id'])
fingerprint_db.execute(query)
query = tables.meta.select().where(
tables.meta.c.id == track_meta['meta_id'])
meta = fingerprint_db.execute(query).fetchone()
assert_equals(None, meta['gid'])
query = tables.submission_result.update().values({
'meta_gid': None
}).where(tables.submission_result.c.meta_id == track_meta['meta_id'])
ingest_db.execute(query)
last_meta_id = backfill_meta_gid(fingerprint_db, ingest_db, 0, 100)
assert_equals(3, last_meta_id)
query = tables.meta.select().where(
tables.meta.c.id == track_meta['meta_id'])
meta = fingerprint_db.execute(query).fetchone()
assert_equals(uuid.UUID('da570fc1-ecfd-5fcd-86d7-009daa0f79e5'),
meta['gid'])
def test_insert_submission(conn):
id = insert_submission(conn, {
'fingerprint': [1,2,3,4,5,6],
'length': 123,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
})
assert_equals(1, id)
rows = conn.execute("""
SELECT fingerprint, length, bitrate, format_id
FROM submission WHERE id=%s
""", (id,)).fetchall()
expected_rows = [
([1,2,3,4,5,6], 123, 192, 1),
]
assert_equals(expected_rows, rows)
def test_import_submission_reuse_fingerprint_100(conn):
prepare_database(conn, """
INSERT INTO fingerprint (fingerprint, length, track_id, submission_count)
VALUES (%(fp)s, %(len)s, 1, 1);
""", dict(fp=TEST_1A_FP_RAW, len=TEST_1A_LENGTH))
id = insert_submission(conn, {
'fingerprint': TEST_1A_FP_RAW,
'length': TEST_1A_LENGTH,
'source_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(conn, submission)
assert_equals(1, fingerprint['id'])
assert_equals(1, fingerprint['track_id'])
def test_import_submission_new_track_different(conn):
prepare_database(conn, """
INSERT INTO fingerprint (fingerprint, length, track_id, submission_count)
VALUES (%(fp)s, %(len)s, 1, 1);
""", dict(fp=TEST_1A_FP_RAW, len=TEST_1A_LENGTH))
id = insert_submission(conn, {
'fingerprint': TEST_2_FP_RAW,
'length': TEST_2_LENGTH,
'source_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(conn, submission)
assert_equals(2, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
def test_import_submission_with_foreignid(conn):
prepare_database(conn, """
INSERT INTO foreignid_vendor (id, name) VALUES (1, 'foo');
INSERT INTO foreignid (id, vendor_id, name) VALUES (1, 1, '123');
""")
id = insert_submission(conn, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'foreignid_id': 1,
})
query = tables.submission.select(tables.submission.c.id == id)
submission = conn.execute(query).fetchone()
fingerprint = import_submission(conn, submission)
query = tables.track_foreignid.select(tables.track_foreignid.c.track_id == fingerprint['track_id'])
track_foreignid = conn.execute(query).fetchone()
assert_equals(1, track_foreignid['submission_count'])
def test_insert_submission(ctx):
# type: (ScriptContext) -> None
ingest_db = ctx.db.get_ingest_db()
id = insert_submission(
ingest_db, {
'fingerprint': [1, 2, 3, 4, 5, 6],
'length': 123,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
})
assert_equals(1, id)
rows = ingest_db.execute(
"""
SELECT fingerprint, length, bitrate, format_id
FROM submission WHERE id=%s
""", (id, )).fetchall()
expected_rows = [
([1, 2, 3, 4, 5, 6], 123, 192, 1),
]
assert_equals(expected_rows, rows)
def test_import_queued_submissions(ctx):
# type: (ScriptContext) -> None
ingest_db = ctx.db.get_ingest_db()
app_db = ctx.db.get_app_db()
fingerprint_db = ctx.db.get_fingerprint_db()
insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'meta': {
'track': 'Foo'
},
})
insert_submission(
ingest_db, {
'fingerprint': TEST_2_FP_RAW,
'length': TEST_2_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
})
insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
})
import_queued_submissions(ingest_db, app_db, fingerprint_db, ctx.index)
count = fingerprint_db.execute(
"SELECT count(*) FROM fingerprint WHERE id IN (1,2,3)").scalar()
assert_equals(2, count)
count = fingerprint_db.execute(
"SELECT count(*) FROM track WHERE id IN (5,6,7)").scalar()
assert_equals(2, count)
def _handle_internal(self, params):
# type: (APIHandlerParams) -> Dict[str, Any]
assert isinstance(params, SubmitHandlerParams)
response = {'submissions': []} # type: Dict[str, Any]
ids = set() # type: Set[int]
fingerprint_db = self.ctx.db.get_fingerprint_db()
ingest_db = self.ctx.db.get_ingest_db()
for p in params.submissions:
mbids = p['mbids'] or [None]
for mbid in mbids:
values = {
'mbid': mbid or None,
'puid': p['puid'] or None,
'bitrate': p['bitrate'] or None,
'fingerprint': p['fingerprint'],
'length': p['duration'],
'format': p['format'] or None,
'account_id': params.account_id,
'application_id': params.application_id,
'application_version': params.application_version,
}
meta_values = dict((n, p[n]) for n in self.meta_fields if p[n])
if any(meta_values.values()):
values['meta'] = meta_values
if p['foreignid']:
values['foreignid'] = p['foreignid']
id = insert_submission(ingest_db, values)
ids.add(id)
submission = {'id': id, 'status': 'pending'}
if p['index']:
submission['index'] = p['index']
response['submissions'].append(submission)
self.ctx.db.session.flush()
self.ctx.db.session.commit()
self.ctx.redis.publish('channel.submissions', json.dumps(list(ids)))
ingest_db = self.ctx.db.get_ingest_db()
fingerprint_db = self.ctx.db.get_fingerprint_db()
clients_waiting_key = 'submission.waiting'
clients_waiting = self.ctx.redis.incr(clients_waiting_key) - 1
try:
max_wait = 10
self.ctx.redis.expire(clients_waiting_key, max_wait)
remaining = min(max(0, max_wait - 2**clients_waiting), params.wait)
logger.debug('starting to wait at %f %d', remaining,
clients_waiting)
while remaining > 0 and ids:
logger.debug('waiting %f seconds', remaining)
time.sleep(0.5) # XXX replace with LISTEN/NOTIFY
remaining -= 0.5
tracks = lookup_submission_status(ingest_db, fingerprint_db,
ids)
if not tracks:
continue
for submission in response['submissions']:
submission_id = submission['id']
assert isinstance(submission_id, int)
track_gid = tracks.get(submission_id)
if track_gid is not None:
submission['status'] = 'imported'
submission['result'] = {'id': track_gid}
ids.remove(submission_id)
finally:
self.ctx.redis.decr(clients_waiting_key)
return response
def _handle_internal(self, params):
response = {'submissions': []}
ids = set()
with self.conn.begin():
source_id = find_or_insert_source(self.conn, params.application_id, params.account_id, params.application_version)
format_ids = {}
for p in params.submissions:
if p['format']:
if p['format'] not in format_ids:
format_ids[p['format']] = find_or_insert_format(self.conn, p['format'])
p['format_id'] = format_ids[p['format']]
for p in params.submissions:
mbids = p['mbids'] or [None]
for mbid in mbids:
values = {
'mbid': mbid or None,
'puid': p['puid'] or None,
'bitrate': p['bitrate'] or None,
'fingerprint': p['fingerprint'],
'length': p['duration'],
'format_id': p.get('format_id'),
'source_id': source_id
}
meta_values = dict((n, p[n] or None) for n in self.meta_fields)
if any(meta_values.itervalues()):
values['meta_id'] = insert_meta(self.conn, meta_values)
if p['foreignid']:
values['foreignid_id'] = find_or_insert_foreignid(self.conn, p['foreignid'])
id = insert_submission(self.conn, values)
ids.add(id)
submission = {'id': id, 'status': 'pending'}
if p['index']:
submission['index'] = p['index']
response['submissions'].append(submission)
if self.redis is not None:
self.redis.publish('channel.submissions', json.dumps(list(ids)))
clients_waiting_key = 'submission.waiting'
clients_waiting = self.redis.incr(clients_waiting_key) - 1
try:
max_wait = 10
self.redis.expire(clients_waiting_key, max_wait)
tracks = {}
remaining = min(max(0, max_wait - 2 ** clients_waiting), params.wait)
logger.debug('starting to wait at %f %d', remaining, clients_waiting)
while remaining > 0 and ids:
logger.debug('waiting %f seconds', remaining)
time.sleep(0.5) # XXX replace with LISTEN/NOTIFY
remaining -= 0.5
tracks = lookup_submission_status(self.conn, ids)
if not tracks:
continue
for submission in response['submissions']:
id = submission['id']
track_gid = tracks.get(id)
if track_gid is not None:
submission['status'] = 'imported'
submission['result'] = {'id': track_gid}
ids.remove(id)
finally:
self.redis.decr(clients_waiting_key)
return response
def _handle_internal(self, params):
response = {'submissions': []}
ids = set()
with self.conn.begin():
source_id = find_or_insert_source(self.conn, params.application_id,
params.account_id,
params.application_version)
format_ids = {}
for p in params.submissions:
if p['format']:
if p['format'] not in format_ids:
format_ids[p['format']] = find_or_insert_format(
self.conn, p['format'])
p['format_id'] = format_ids[p['format']]
for p in params.submissions:
mbids = p['mbids'] or [None]
for mbid in mbids:
values = {
'mbid': mbid or None,
'puid': p['puid'] or None,
'bitrate': p['bitrate'] or None,
'fingerprint': p['fingerprint'],
'length': p['duration'],
'format_id': p.get('format_id'),
'source_id': source_id
}
meta_values = dict(
(n, p[n] or None) for n in self.meta_fields)
if any(meta_values.itervalues()):
values['meta_id'] = insert_meta(self.conn, meta_values)
if p['foreignid']:
values['foreignid_id'] = find_or_insert_foreignid(
self.conn, p['foreignid'])
id = insert_submission(self.conn, values)
ids.add(id)
submission = {'id': id, 'status': 'pending'}
if p['index']:
submission['index'] = p['index']
response['submissions'].append(submission)
if self.redis is not None:
self.redis.publish('channel.submissions', json.dumps(list(ids)))
clients_waiting_key = 'submission.waiting'
clients_waiting = self.redis.incr(clients_waiting_key) - 1
try:
max_wait = 10
self.redis.expire(clients_waiting_key, max_wait)
tracks = {}
remaining = min(max(0, max_wait - 2**clients_waiting), params.wait)
logger.debug('starting to wait at %f %d', remaining,
clients_waiting)
while remaining > 0 and ids:
logger.debug('waiting %f seconds', remaining)
time.sleep(0.5) # XXX replace with LISTEN/NOTIFY
remaining -= 0.5
tracks = lookup_submission_status(self.conn, ids)
if not tracks:
continue
for submission in response['submissions']:
id = submission['id']
track_gid = tracks.get(id)
if track_gid is not None:
submission['status'] = 'imported'
submission['result'] = {'id': track_gid}
ids.remove(id)
finally:
self.redis.decr(clients_waiting_key)
return response
def test_import_submission(ctx):
# type: (ScriptContext) -> None
ingest_db = ctx.db.get_ingest_db()
app_db = ctx.db.get_app_db()
fingerprint_db = ctx.db.get_fingerprint_db()
# first submission
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'account_id': 1,
'application_id': 1,
'format': 'FLAC',
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
assert_equals(1, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_true(submission['handled'])
query = tables.track_mbid.select(
tables.track_mbid.c.track_id == fingerprint['track_id'])
track_mbid = fingerprint_db.execute(query).fetchone()
assert_equals(1, track_mbid['submission_count'])
query = tables.track_puid.select(
tables.track_puid.c.track_id == fingerprint['track_id'])
track_puid = fingerprint_db.execute(query).fetchone()
assert_equals(1, track_puid['submission_count'])
query = tables.fingerprint.select(
tables.fingerprint.c.id == fingerprint['id'])
fingerprint = fingerprint_db.execute(query).fetchone()
assert fingerprint is not None
assert_equals(1, fingerprint['submission_count'])
assert_equals(1, fingerprint['format_id'])
# second submission
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_2_FP_RAW,
'length': TEST_2_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
assert_equals(2, fingerprint['id'])
assert_equals(6, fingerprint['track_id'])
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_true(submission['handled'])
# third submission (same as the first one)
submission_id = insert_submission(
ingest_db, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'mbid': '1f143d2b-db04-47cc-82a0-eee6efaa1142',
'puid': '7c1c6753-c834-44b1-884a-a5166c093139',
})
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_false(submission['handled'])
fingerprint = import_submission(ingest_db, app_db, fingerprint_db,
ctx.index, submission)
assert fingerprint is not None
assert_equals(1, fingerprint['id'])
assert_equals(5, fingerprint['track_id'])
query = tables.submission.select(tables.submission.c.id == submission_id)
submission = ingest_db.execute(query).fetchone()
assert_true(submission['handled'])
query = tables.track_mbid.select(
tables.track_mbid.c.track_id == fingerprint['track_id'])
track_mbid = fingerprint_db.execute(query).fetchone()
assert_equals(2, track_mbid['submission_count'])
query = tables.track_puid.select(
tables.track_puid.c.track_id == fingerprint['track_id'])
track_puid = fingerprint_db.execute(query).fetchone()
assert_equals(2, track_puid['submission_count'])
query = tables.fingerprint.select(
tables.fingerprint.c.id == fingerprint['id'])
fingerprint = fingerprint_db.execute(query).fetchone()
assert fingerprint is not None
assert_equals(2, fingerprint['submission_count'])
