def test_import_queued_submissions(conn):
insert_meta(conn, {'track': 'Foo'})
insert_submission(conn, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
'meta_id': 1,
})
insert_submission(conn, {
'fingerprint': TEST_2_FP_RAW,
'length': TEST_2_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
})
insert_submission(conn, {
'fingerprint': TEST_1_FP_RAW,
'length': TEST_1_LENGTH,
'bitrate': 192,
'source_id': 1,
'format_id': 1,
})
import_queued_submissions(conn)
count = conn.execute("SELECT count(*) FROM fingerprint WHERE id IN (1,2,3)").scalar()
assert_equals(2, count)
count = conn.execute("SELECT count(*) FROM track WHERE id IN (5,6,7)").scalar()
assert_equals(2, count)
def _handle_internal(self, params):
with self.conn.begin():
source_id = find_or_insert_source(self.conn, params.application_id, params.account_id, params.application_version)
format_ids = {}
for p in params.submissions:
if p['format']:
if p['format'] not in format_ids:
format_ids[p['format']] = find_or_insert_format(self.conn, p['format'])
p['format_id'] = format_ids[p['format']]
for p in params.submissions:
mbids = p['mbids'] or [None]
for mbid in mbids:
values = {
'mbid': mbid or None,
'puid': p['puid'] or None,
'bitrate': p['bitrate'] or None,
'fingerprint': p['fingerprint'],
'length': p['duration'],
'format_id': p.get('format_id'),
'source_id': source_id
}
meta_values = dict((n, p[n] or None) for n in self.meta_fields)
if any(meta_values.itervalues()):
values['meta_id'] = insert_meta(self.conn, meta_values)
if p['foreignid']:
values['foreignid_id'] = find_or_insert_foreignid(self.conn, p['foreignid'])
insert_submission(self.conn, values)
return {}
def test_insert_meta(ctx):
# type: (ScriptContext) -> None
meta_id, meta_gid = insert_meta(ctx.db.get_fingerprint_db(), {
'track': 'Voodoo People',
'artist': 'The Prodigy',
'album': 'Music For The Jitled People',
'album_artist': 'Prodigy',
'track_no': 2,
'disc_no': 3,
'year': 2030
})
assert_equals(3, meta_id)
assert_equals(uuid.UUID('398d828b-b601-5c58-a135-d5c81116da7c'), meta_gid)
row = dict(ctx.db.get_fingerprint_db().execute("SELECT * FROM meta WHERE id=%s", meta_id).fetchone())
expected = {
'id': meta_id,
'track': 'Voodoo People',
'artist': 'The Prodigy',
'album': 'Music For The Jitled People',
'album_artist': 'Prodigy',
'track_no': 2,
'disc_no': 3,
'year': 2030,
'gid': uuid.UUID('398d828b-b601-5c58-a135-d5c81116da7c'),
}
assert row['created'] is not None
del row['created']
assert_equals(expected, row)
def _handle_internal(self, params):
with self.conn.begin():
source_id = find_or_insert_source(
self.conn, params.application_id, params.account_id, params.application_version
)
format_ids = {}
for p in params.submissions:
if p["format"]:
if p["format"] not in format_ids:
format_ids[p["format"]] = find_or_insert_format(self.conn, p["format"])
p["format_id"] = format_ids[p["format"]]
for p in params.submissions:
mbids = p["mbids"] or [None]
for mbid in mbids:
values = {
"mbid": mbid or None,
"puid": p["puid"] or None,
"bitrate": p["bitrate"] or None,
"fingerprint": p["fingerprint"],
"length": p["duration"],
"format_id": p.get("format_id"),
"source_id": source_id,
}
meta_values = dict((n, p[n] or None) for n in self.meta_fields)
if any(meta_values.itervalues()):
values["meta_id"] = insert_meta(self.conn, meta_values)
if p["foreignid"]:
values["foreignid_id"] = find_or_insert_foreignid(self.conn, p["foreignid"])
insert_submission(self.conn, values)
return {}
def test_insert_meta(conn):
id = insert_meta(conn, {
'track': 'Voodoo People',
'artist': 'The Prodigy',
'album': 'Music For The Jitled People',
'album_artist': 'Prodigy',
'track_no': 2,
'disc_no': 3,
'year': 2030
})
assert_equals(3, id)
row = conn.execute("SELECT * FROM meta WHERE id=%s", id).fetchone()
expected = {
'id': id,
'track': 'Voodoo People',
'artist': 'The Prodigy',
'album': 'Music For The Jitled People',
'album_artist': 'Prodigy',
'track_no': 2,
'disc_no': 3,
'year': 2030
}
assert_equals(expected, dict(row))
def test_insert_meta(conn):
id = insert_meta(conn, {
'track': 'Voodoo People',
'artist': 'The Prodigy',
'album': 'Music For The Jitled People',
'album_artist': 'Prodigy',
'track_no': 2,
'disc_no': 3,
'year': 2030
})
assert_equals(1, id)
row = conn.execute("SELECT * FROM meta WHERE id=1").fetchone()
expected = {
'id': 1,
'track': 'Voodoo People',
'artist': 'The Prodigy',
'album': 'Music For The Jitled People',
'album_artist': 'Prodigy',
'track_no': 2,
'disc_no': 3,
'year': 2030
}
assert_equals(expected, dict(row))
def _handle_internal(self, params):
response = {'submissions': []}
ids = set()
with self.conn.begin():
source_id = find_or_insert_source(self.conn, params.application_id,
params.account_id,
params.application_version)
format_ids = {}
for p in params.submissions:
if p['format']:
if p['format'] not in format_ids:
format_ids[p['format']] = find_or_insert_format(
self.conn, p['format'])
p['format_id'] = format_ids[p['format']]
for p in params.submissions:
mbids = p['mbids'] or [None]
for mbid in mbids:
values = {
'mbid': mbid or None,
'puid': p['puid'] or None,
'bitrate': p['bitrate'] or None,
'fingerprint': p['fingerprint'],
'length': p['duration'],
'format_id': p.get('format_id'),
'source_id': source_id
}
meta_values = dict(
(n, p[n] or None) for n in self.meta_fields)
if any(meta_values.itervalues()):
values['meta_id'] = insert_meta(self.conn, meta_values)
if p['foreignid']:
values['foreignid_id'] = find_or_insert_foreignid(
self.conn, p['foreignid'])
id = insert_submission(self.conn, values)
ids.add(id)
submission = {'id': id, 'status': 'pending'}
if p['index']:
submission['index'] = p['index']
response['submissions'].append(submission)
if self.redis is not None:
self.redis.publish('channel.submissions', json.dumps(list(ids)))
clients_waiting_key = 'submission.waiting'
clients_waiting = self.redis.incr(clients_waiting_key) - 1
try:
max_wait = 10
self.redis.expire(clients_waiting_key, max_wait)
tracks = {}
remaining = min(max(0, max_wait - 2**clients_waiting), params.wait)
logger.debug('starting to wait at %f %d', remaining,
clients_waiting)
while remaining > 0 and ids:
logger.debug('waiting %f seconds', remaining)
time.sleep(0.5) # XXX replace with LISTEN/NOTIFY
remaining -= 0.5
tracks = lookup_submission_status(self.conn, ids)
if not tracks:
continue
for submission in response['submissions']:
id = submission['id']
track_gid = tracks.get(id)
if track_gid is not None:
submission['status'] = 'imported'
submission['result'] = {'id': track_gid}
ids.remove(id)
finally:
self.redis.decr(clients_waiting_key)
return response
def _handle_internal(self, params):
response = {'submissions': []}
ids = set()
with self.conn.begin():
source_id = find_or_insert_source(self.conn, params.application_id, params.account_id, params.application_version)
format_ids = {}
for p in params.submissions:
if p['format']:
if p['format'] not in format_ids:
format_ids[p['format']] = find_or_insert_format(self.conn, p['format'])
p['format_id'] = format_ids[p['format']]
for p in params.submissions:
mbids = p['mbids'] or [None]
for mbid in mbids:
values = {
'mbid': mbid or None,
'puid': p['puid'] or None,
'bitrate': p['bitrate'] or None,
'fingerprint': p['fingerprint'],
'length': p['duration'],
'format_id': p.get('format_id'),
'source_id': source_id
}
meta_values = dict((n, p[n] or None) for n in self.meta_fields)
if any(meta_values.itervalues()):
values['meta_id'] = insert_meta(self.conn, meta_values)
if p['foreignid']:
values['foreignid_id'] = find_or_insert_foreignid(self.conn, p['foreignid'])
id = insert_submission(self.conn, values)
ids.add(id)
submission = {'id': id, 'status': 'pending'}
if p['index']:
submission['index'] = p['index']
response['submissions'].append(submission)
if self.redis is not None:
self.redis.publish('channel.submissions', json.dumps(list(ids)))
clients_waiting_key = 'submission.waiting'
clients_waiting = self.redis.incr(clients_waiting_key) - 1
try:
max_wait = 10
self.redis.expire(clients_waiting_key, max_wait)
tracks = {}
remaining = min(max(0, max_wait - 2 ** clients_waiting), params.wait)
logger.debug('starting to wait at %f %d', remaining, clients_waiting)
while remaining > 0 and ids:
logger.debug('waiting %f seconds', remaining)
time.sleep(0.5) # XXX replace with LISTEN/NOTIFY
remaining -= 0.5
tracks = lookup_submission_status(self.conn, ids)
if not tracks:
continue
for submission in response['submissions']:
id = submission['id']
track_gid = tracks.get(id)
if track_gid is not None:
submission['status'] = 'imported'
submission['result'] = {'id': track_gid}
ids.remove(id)
finally:
self.redis.decr(clients_waiting_key)
return response
