def find_track_duplicates(conn, fingerprint, index=None):
with conn.begin():
searcher = FingerprintSearcher(conn, index)
searcher.min_score = const.TRACK_MERGE_THRESHOLD
matches = searcher.search(fingerprint['fingerprint'], fingerprint['length'])
if not matches:
logger.debug("Not matched itself!")
return
logged = False
match = matches[0]
all_track_ids = set()
possible_track_ids = set()
for m in matches:
if m['track_id'] in all_track_ids:
continue
all_track_ids.add(m['track_id'])
if can_add_fp_to_track(conn, m['track_id'], fingerprint['fingerprint'], fingerprint['length']):
if m['id'] != fingerprint['id']:
if not logged:
logger.debug("Deduplicating fingerprint %d", fingerprint['id'])
logged = True
logger.debug("Fingerprint %d with track %d is %d%% similar", m['id'], m['track_id'], m['score'] * 100)
possible_track_ids.add(m['track_id'])
if len(possible_track_ids) > 1:
for group in can_merge_tracks(conn, possible_track_ids):
if len(group) > 1:
target_track_id = min(group)
group.remove(target_track_id)
#logger.debug("Would like to merge tracks %r into %d", list(group), target_track_id)
merge_tracks(conn, target_track_id, list(group))
#raise Exception(1)
break
conn.execute("INSERT INTO fingerprint_deduplicate (id) VALUES (%s)", fingerprint['id'])
def find_track_duplicates(conn, fingerprint, index=None):
with conn.begin():
searcher = FingerprintSearcher(conn, index)
searcher.min_score = const.TRACK_MERGE_THRESHOLD
matches = searcher.search(fingerprint['fingerprint'], fingerprint['length'])
if not matches:
logger.debug("Not matched itself!")
return
logged = False
all_track_ids = set()
possible_track_ids = set()
for m in matches:
if m['track_id'] in all_track_ids:
continue
all_track_ids.add(m['track_id'])
if can_add_fp_to_track(conn, m['track_id'], fingerprint['fingerprint'], fingerprint['length']):
if m['id'] != fingerprint['id']:
if not logged:
logger.debug("Deduplicating fingerprint %d", fingerprint['id'])
logged = True
logger.debug("Fingerprint %d with track %d is %d%% similar", m['id'], m['track_id'], m['score'] * 100)
possible_track_ids.add(m['track_id'])
if len(possible_track_ids) > 1:
for group in can_merge_tracks(conn, possible_track_ids):
if len(group) > 1:
target_track_id = min(group)
group.remove(target_track_id)
merge_tracks(conn, target_track_id, list(group))
break
conn.execute("INSERT INTO fingerprint_deduplicate (id) VALUES (%s)", fingerprint['id'])
def import_submission(ingest_db, app_db, fingerprint_db, index_pool,
submission):
# type: (IngestDB, AppDB, FingerprintDB, IndexClientPool, Dict[str, Any]) -> Optional[Dict[str, Any]]
"""
Import the given submission into the main fingerprint database
"""
handled_at = datetime.datetime.now(pytz.utc)
update_stmt = schema.submission.update().where(
schema.submission.c.id == submission['id'])
ingest_db.execute(update_stmt.values(handled=True))
ingest_db.execute(update_stmt.values(handled=True, handled_at=handled_at))
logger.info("Importing submission %d with MBIDs %s", submission['id'],
submission['mbid'])
num_unique_items = len(set(submission['fingerprint']))
if num_unique_items < const.FINGERPRINT_MIN_UNIQUE_ITEMS:
logger.info("Skipping, has only %d unique items", num_unique_items)
return None
num_query_items = fingerprint_db.execute(
"SELECT icount(acoustid_extract_query(%(fp)s))",
dict(fp=submission['fingerprint']))
if not num_query_items:
logger.info("Skipping, no data to index")
return None
source_id = submission['source_id']
if source_id is not None:
source = get_source(app_db, source_id)
if source is None:
logger.error("Source not found")
return None
else:
source = {
'application_id': submission['application_id'],
'version': submission['application_version'],
'account_id': submission['account_id'],
}
source_id = find_or_insert_source(app_db, source['application_id'],
source['account_id'],
source['version'])
submission_result = {
'submission_id': submission['id'],
'created': submission['created'],
'handled_at': handled_at,
'account_id': source['account_id'],
'application_id': source['application_id'],
'application_version': source['version'],
}
format_id = submission['format_id']
if format_id is None and submission['format'] is not None:
format_id = find_or_insert_format(app_db, submission['format'])
fingerprint = {
'id': None,
'track_id': None,
'fingerprint': submission['fingerprint'],
'length': submission['length'],
'bitrate': submission['bitrate'],
'format_id': format_id,
}
searcher = FingerprintSearcher(fingerprint_db, index_pool, fast=False)
searcher.min_score = const.TRACK_MERGE_THRESHOLD
matches = searcher.search(submission['fingerprint'], submission['length'])
if matches:
all_track_ids = set() # type: Set[int]
possible_track_ids = set() # type: Set[int]
for m in matches:
if m.track_id in all_track_ids:
continue
all_track_ids.add(m.track_id)
logger.debug("Fingerprint %d with track %d is %d%% similar",
m.fingerprint_id, m.track_id, m.score * 100)
if can_add_fp_to_track(fingerprint_db, m.track_id,
submission['fingerprint'],
submission['length']):
possible_track_ids.add(m.track_id)
if not fingerprint['track_id']:
fingerprint['track_id'] = m.track_id
if m.score > const.FINGERPRINT_MERGE_THRESHOLD:
fingerprint['id'] = m.fingerprint_id
if len(possible_track_ids) > 1:
for group in can_merge_tracks(fingerprint_db, possible_track_ids):
if fingerprint['track_id'] in group and len(group) > 1:
fingerprint['track_id'] = min(group)
group.remove(fingerprint['track_id'])
merge_tracks(fingerprint_db, ingest_db,
fingerprint['track_id'], list(group))
break
if not fingerprint['track_id']:
fingerprint['track_id'] = insert_track(fingerprint_db)
assert isinstance(fingerprint['track_id'], int)
submission_result['track_id'] = fingerprint['track_id']
if not fingerprint['id']:
fingerprint['id'] = insert_fingerprint(fingerprint_db, ingest_db,
fingerprint, submission['id'],
source_id)
else:
assert isinstance(fingerprint['id'], int)
inc_fingerprint_submission_count(fingerprint_db, ingest_db,
fingerprint['id'], submission['id'],
source_id)
submission_result['fingerprint_id'] = fingerprint['id']
if submission['mbid'] and submission[
'mbid'] != '00000000-0000-0000-0000-000000000000':
insert_mbid(fingerprint_db, ingest_db, fingerprint['track_id'],
submission['mbid'], submission['id'], source_id)
submission_result['mbid'] = submission['mbid']
if submission['puid'] and submission[
'puid'] != '00000000-0000-0000-0000-000000000000':
insert_puid(fingerprint_db, ingest_db, fingerprint['track_id'],
submission['puid'], submission['id'], source_id)
submission_result['puid'] = submission['puid']
if submission['meta_id'] or submission['meta']:
meta_id = submission['meta_id'] # type: Optional[int]
meta_gid = None # type: Optional[uuid.UUID]
if meta_id is None:
meta_id, meta_gid = find_or_insert_meta(fingerprint_db,
submission['meta'])
else:
found, meta_gid = check_meta_id(fingerprint_db, meta_id)
if not found:
logger.error("Meta not found")
meta_id = None
if meta_id is not None:
insert_track_meta(fingerprint_db, ingest_db,
fingerprint['track_id'], meta_id,
submission['id'], source_id)
submission_result['meta_id'] = meta_id
submission_result['meta_gid'] = meta_gid
if submission['foreignid_id'] or submission['foreignid']:
foreignid_id = submission['foreignid_id']
if foreignid_id is None:
foreignid = submission['foreignid']
foreignid_id = find_or_insert_foreignid(fingerprint_db, foreignid)
else:
foreignid = get_foreignid(fingerprint_db, foreignid_id)
insert_track_foreignid(fingerprint_db, ingest_db,
fingerprint['track_id'], foreignid_id,
submission['id'], source_id)
submission_result['foreignid'] = foreignid
insert_submission_result(ingest_db, submission_result)
return fingerprint
def import_submission(conn, submission, index=None):
"""
Import the given submission into the main fingerprint database
"""
with conn.begin():
update_stmt = schema.submission.update().where(
schema.submission.c.id == submission['id'])
conn.execute(update_stmt.values(handled=True))
mbids = []
if submission['mbid']:
mbids.append(resolve_mbid_redirect(conn, submission['mbid']))
if submission['puid']:
min_duration = submission['length'] - 15
max_duration = submission['length'] + 15
mbids.extend(find_puid_mbids(conn, submission['puid'], min_duration, max_duration))
logger.info("Importing submission %d with MBIDs %s",
submission['id'], ', '.join(mbids))
num_unique_items = len(set(submission['fingerprint']))
if num_unique_items < const.FINGERPRINT_MIN_UNIQUE_ITEMS:
logger.info("Skipping, has only %d unique items", num_unique_items)
return
num_query_items = conn.execute("SELECT icount(acoustid_extract_query(%(fp)s))", dict(fp=submission['fingerprint']))
if not num_query_items:
logger.info("Skipping, no data to index")
return
searcher = FingerprintSearcher(conn, index, fast=False)
searcher.min_score = const.TRACK_MERGE_THRESHOLD
matches = searcher.search(submission['fingerprint'], submission['length'])
fingerprint = {
'id': None,
'track_id': None,
'fingerprint': submission['fingerprint'],
'length': submission['length'],
'bitrate': submission['bitrate'],
'format_id': submission['format_id'],
}
if matches:
match = matches[0]
all_track_ids = set()
possible_track_ids = set()
for m in matches:
if m['track_id'] in all_track_ids:
continue
all_track_ids.add(m['track_id'])
logger.debug("Fingerprint %d with track %d is %d%% similar", m['id'], m['track_id'], m['score'] * 100)
if can_add_fp_to_track(conn, m['track_id'], submission['fingerprint'], submission['length']):
possible_track_ids.add(m['track_id'])
if not fingerprint['track_id']:
fingerprint['track_id'] = m['track_id']
if m['score'] > const.FINGERPRINT_MERGE_THRESHOLD:
fingerprint['id'] = m['id']
if len(possible_track_ids) > 1:
for group in can_merge_tracks(conn, possible_track_ids):
if fingerprint['track_id'] in group and len(group) > 1:
fingerprint['track_id'] = min(group)
group.remove(fingerprint['track_id'])
merge_tracks(conn, fingerprint['track_id'], list(group))
break
if not fingerprint['track_id']:
fingerprint['track_id'] = insert_track(conn)
if not fingerprint['id']:
fingerprint['id'] = insert_fingerprint(conn, fingerprint, submission['id'], submission['source_id'])
else:
inc_fingerprint_submission_count(conn, fingerprint['id'], submission['id'], submission['source_id'])
for mbid in mbids:
insert_mbid(conn, fingerprint['track_id'], mbid, submission['id'], submission['source_id'])
if submission['puid'] and submission['puid'] != '00000000-0000-0000-0000-000000000000':
insert_puid(conn, fingerprint['track_id'], submission['puid'], submission['id'], submission['source_id'])
if submission['meta_id']:
insert_track_meta(conn, fingerprint['track_id'], submission['meta_id'], submission['id'], submission['source_id'])
if submission['foreignid_id']:
insert_track_foreignid(conn, fingerprint['track_id'], submission['foreignid_id'], submission['id'], submission['source_id'])
return fingerprint
def import_submission(conn, submission, index=None):
"""
Import the given submission into the main fingerprint database
"""
with conn.begin():
update_stmt = schema.submission.update().where(
schema.submission.c.id == submission['id'])
conn.execute(update_stmt.values(handled=True))
mbids = []
if submission['mbid']:
mbids.append(resolve_mbid_redirect(conn, submission['mbid']))
logger.info("Importing submission %d with MBIDs %s", submission['id'],
', '.join(mbids))
num_unique_items = len(set(submission['fingerprint']))
if num_unique_items < const.FINGERPRINT_MIN_UNIQUE_ITEMS:
logger.info("Skipping, has only %d unique items", num_unique_items)
return
num_query_items = conn.execute(
"SELECT icount(acoustid_extract_query(%(fp)s))",
dict(fp=submission['fingerprint']))
if not num_query_items:
logger.info("Skipping, no data to index")
return
searcher = FingerprintSearcher(conn, index, fast=False)
searcher.min_score = const.TRACK_MERGE_THRESHOLD
matches = searcher.search(submission['fingerprint'],
submission['length'])
fingerprint = {
'id': None,
'track_id': None,
'fingerprint': submission['fingerprint'],
'length': submission['length'],
'bitrate': submission['bitrate'],
'format_id': submission['format_id'],
}
if matches:
all_track_ids = set()
possible_track_ids = set()
for m in matches:
if m['track_id'] in all_track_ids:
continue
all_track_ids.add(m['track_id'])
logger.debug("Fingerprint %d with track %d is %d%% similar",
m['id'], m['track_id'], m['score'] * 100)
if can_add_fp_to_track(conn, m['track_id'],
submission['fingerprint'],
submission['length']):
possible_track_ids.add(m['track_id'])
if not fingerprint['track_id']:
fingerprint['track_id'] = m['track_id']
if m['score'] > const.FINGERPRINT_MERGE_THRESHOLD:
fingerprint['id'] = m['id']
if len(possible_track_ids) > 1:
for group in can_merge_tracks(conn, possible_track_ids):
if fingerprint['track_id'] in group and len(group) > 1:
fingerprint['track_id'] = min(group)
group.remove(fingerprint['track_id'])
merge_tracks(conn, fingerprint['track_id'],
list(group))
break
if not fingerprint['track_id']:
fingerprint['track_id'] = insert_track(conn)
if not fingerprint['id']:
fingerprint['id'] = insert_fingerprint(conn, fingerprint,
submission['id'],
submission['source_id'])
else:
inc_fingerprint_submission_count(conn, fingerprint['id'],
submission['id'],
submission['source_id'])
for mbid in mbids:
insert_mbid(conn, fingerprint['track_id'], mbid, submission['id'],
submission['source_id'])
if submission['puid'] and submission[
'puid'] != '00000000-0000-0000-0000-000000000000':
insert_puid(conn, fingerprint['track_id'], submission['puid'],
submission['id'], submission['source_id'])
if submission['meta_id']:
insert_track_meta(conn, fingerprint['track_id'],
submission['meta_id'], submission['id'],
submission['source_id'])
if submission['foreignid_id']:
insert_track_foreignid(conn, fingerprint['track_id'],
submission['foreignid_id'],
submission['id'], submission['source_id'])
return fingerprint