def test_file(filename, local = False, expect_match=True, original_TRID=None, remove_file = True):
"""
Test a single file. This will return a code like tn, tp, fp, err, etc
"""
matching_TRID = None
if filename is None:
return "err-munge" # most likely a munge error (file too short, unreadable, etc)
try:
# don't pass start and duration to codegen, assume the munging takes care of codelength
if not local:
query_obj = song.util.codegen(filename, start=-1, duration=-1)
s = fp.best_match_for_query(query_obj[0]["code"])
if s.TRID is not None:
matching_TRID = s.TRID
else:
query_obj = song.util.codegen(filename, start=-1, duration=-1)
s = fp.best_match_for_query(query_obj[0]["code"], local=local)
if s.TRID is not None:
matching_TRID = s.TRID
except IOError:
print "TIMEOUT from API server"
return "err-api"
except TypeError: # codegen returned none
return "err-codegen"
if remove_file:
if os.path.exists(filename):
os.remove(filename)
else:
return "err-munge"
# if is not None there was a response
if s is not None:
# there was a match
if len(s) > 0:
# if we're expecting a match, check that it's the right one
if expect_match:
if original_TRID == matching_TRID:
# we expected a match, it's the right one. TP
return "tp"
else:
# we expected a match but it's the wrong one. FP-a
return "fp-a"
else:
# we did not expect a match. FP-b
return "fp-b"
else:
# there was no match from the API
if expect_match:
# we expected a match. FN
return "fn"
else:
# we did not expect a match. TN
return "tn"
else:
# s is None, that means API error-- almost definitely codegen returned nothing.
return "err-codegen"
def lookup(FILE):
codes = codegen(FILE)
if len(codes) and "code" in codes[0]:
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
print "Got result:", result
if result.TRID:
#I want to print thr current time and the name of music together
ID = "ID: %s" % (result.TRID)
myFile.write(
ID + str(CurrentTime)
) #+"\n"+"Artist: %s" % (result.metadata.get("artist"))+"\n"+"Song: %s" % (result.metadata.get("track"+"\n"))+"\n")
myFile.write("Artist: %s" % (result.metadata.get("artist")))
myFile.write("Song: %s" % (result.metadata.get("track")) +
"\n")
#print ID+str(CurrentTime)
#print "Artist: %s" % (result.metadata.get("artist"))
#print "Song: %s" % (result.metadata.get("track"+"\n"))
else:
#print "No match. This track may not be in the database yet."
Unmatch = "No match. This track may not be in the database yet."
myFile.write(Unmatch + str(CurrentTime) + "\n")
else:
Undecode = "Couldn't decode" + str(FILE)
myFile.write(Undecode)
def query(request):
mp3 = request.FILES['mp3']
# Convert music -> fingerprint code
fp_code = generate_fingerprint(mp3)
if fp_code is dict:
return HttpResponse(fp_code['error'], status=400)
# First see if this is a compressed code
if re.match('[A-Za-z\/\+\_\-]', fp_code) is not None:
code_string = fp.decode_code_string(fp_code)
if code_string is None:
result = json.dumps({"error": "cannot decode code string %s" % fp_code})
return HttpResponse(result, status=400)
else:
code_string = fp_code
response = fp.best_match_for_query(code_string)
metadata = response.metadata
if metadata:
metadata.pop('import_date')
data = json.dumps({"ok": True, "message": response.message(), "match": response.match(),
"score": response.score, \
"qtime": response.qtime, "track_id": response.TRID,
"total_time": response.total_time,
"metadata": metadata})
return HttpResponse(data, status=200)
def test_single(filename, local=False, **munge_kwargs):
"""
Perform a test on a single file. Prints more diagnostic information than usual.
"""
(new_file, what) = munge(filename, **munge_kwargs)
query_obj = song.util.codegen(new_file, start=-1, duration=-1)
s = fp.best_match_for_query(query_obj[0]["code"], local=local)
if s.TRID is not None:
if local:
metad = _local_bigeval[s.TRID]
else:
metad = fp.metadata_for_track_id(s.TRID)
metad["title"] = metad["track"]
song_metadata = {
"artist": metad.get("artist", ""),
"release": metad.get("release", ""),
"title": metad.get("title", "")
}
print str(song_metadata)
else:
print "No match"
decoded = fp.decode_code_string(query_obj[0]["code"])
print str(len(decoded.split(" ")) / 2) + " codes in original"
response = fp.query_fp(decoded, local=local, rows=15)
if response is not None:
print "From FP flat:"
tracks = {}
scores = {}
for r in response.results:
trid = r["track_id"].split("-")[0]
if local:
metad = _local_bigeval[trid]
else:
metad = fp.metadata_for_track_id(trid)
metad["title"] = metad["track"]
m = {
"artist": metad.get("artist", ""),
"release": metad.get("release", ""),
"title": metad.get("title", "")
}
if m is not None:
actual_match = fp.actual_matches(
decoded, fp.fp_code_for_track_id(r["track_id"],
local=local))
tracks[r["track_id"]] = (m, r["score"], actual_match)
scores[r["track_id"]] = actual_match
else:
print "problem getting metadata for " + r["track_id"]
sorted_scores = sorted(scores.iteritems(),
key=lambda (k, v): (v, k),
reverse=True)
for (trackid, score) in sorted_scores:
(m, score, actual_match) = tracks[trackid]
print trackid + " (" + str(int(score)) + ", " + str(
actual_match) + ") - " + m["artist"] + " - " + m["title"]
else:
print "response from fp flat was None -- decoded code was " + str(
decoded)
os.remove(new_file)
def GET(self):
stuff = web.input(fp_code="")
response = fp.best_match_for_query(stuff.fp_code)
web.header('Content-Type', 'text/plain')
return json.dumps({"ok":True, "query":stuff.fp_code, "message":response.message(), "match":response.match(), "score":response.score, \
"qtime":response.qtime, "track_id":response.TRID, "total_time":response.total_time})
def _query_tag(fp_code, query_obj=None):
"""
Queries the database for given fingerprint
Args:
fp_code: A fingerprint string
query_obj: An optional dict of echoprint-codegen
"""
response = fp.best_match_for_query(fp_code)
print response.message()
if response:
d = {
"ok":True,
"message":response.message(),
"match":response.match(),
"score":response.score,
"qtime":response.qtime,
"track_id":response.TRID,
"total_time":response.total_time
}
if "track" in response.metadata:
d["track"] = response.metadata["track"]
if "artist" in response.metadata:
d["artist"] = response.metadata["artist"]
return json.dumps(d)
def lookup(FILE):
codes = codegen(FILE)
if len(codes) and "code" in codes[0]:
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
print "Got result:", result
if result.TRID:
#I want to print thr current time and the name of music together
ID = "ID: %s" % (result.TRID)
myFile.write(ID+str(CurrentTime))#+"\n"+"Artist: %s" % (result.metadata.get("artist"))+"\n"+"Song: %s" % (result.metadata.get("track"+"\n"))+"\n")
myFile.write("Artist: %s" % (result.metadata.get("artist")))
myFile.write("Song: %s" % (result.metadata.get("track"))+"\n")
#print ID+str(CurrentTime)
#print "Artist: %s" % (result.metadata.get("artist"))
#print "Song: %s" % (result.metadata.get("track"+"\n"))
else:
#print "No match. This track may not be in the database yet."
Unmatch = "No match. This track may not be in the database yet."
myFile.write(Unmatch+str(CurrentTime)+"\n")
else:
Undecode = "Couldn't decode" + str(FILE)
myFile.write(Undecode)
def process_file(filename, t):
codes = codegen(filename)
if codes is None:
return -2
if len(codes) == 0:
return -3
if "code" not in codes[0]:
return -4
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
if result.TRID:
#Melody is recognized
track_id = result.TRID
global last, time
#Insert tracks only once
if (last == 0) or (last != track_id):
last = track_id
mytime = time + datetime.timedelta(seconds=t)
conn = MySQLdb.connect(host="localhost",
user="******",
passwd="ulut123",
db="pymusic",
charset='utf8')
db = conn.cursor()
db.execute(
"""INSERT INTO test_played_melody(track_id,radio,date_played,time_played,radio_id) VALUES (%s,%s,%s,%s,%s)""",
(track_id, radio, date, mytime.time(), radio_id))
conn.commit()
db.close()
conn.close()
def test_single(filename, local=False, **munge_kwargs):
"""
Perform a test on a single file. Prints more diagnostic information than usual.
"""
(new_file, what) = munge(filename, **munge_kwargs)
query_obj = song.util.codegen(new_file, start=-1, duration=-1)
if not local:
s = fp.best_match_for_query(query_obj[0]["code"])
if s.TRID is not None:
metad = _local_bigeval[s.TRID]
song_metadata = {"artist": metad["artist"], "release": metad["release"], "title": metad["title"]}
print str(song_metadata)
else:
print "No match"
else:
s = fp.best_match_for_query(query_obj[0]["code"],local=local)
if s.TRID is not None:
metad = _local_bigeval[s.TRID]
song_metadata = {"artist": metad["artist"], "release": metad["release"], "title": metad["title"]}
print str(song_metadata)
else:
print "No match"
decoded = fp.decode_code_string(query_obj[0]["code"])
print str(len(decoded.split(" "))/2) + " codes in original"
response = fp.query_fp(decoded, local=local, rows=15)
if response is not None:
print "From FP flat:"
tracks = {}
scores = {}
for r in response.results:
trid = r["track_id"].split("-")[0]
metad = _local_bigeval[trid]
m = {"artist": metad["artist"], "release": metad["release"], "title": metad["title"]}
if m is not None:
actual_match = fp.actual_matches(decoded, fp.fp_code_for_track_id(r["track_id"], local=local))
tracks[r["track_id"]] = (m, r["score"], actual_match)
scores[r["track_id"]] = actual_match
else:
print "problem getting metadata for " + r["track_id"]
sorted_scores = sorted(scores.iteritems(), key=lambda (k,v): (v,k), reverse=True)
for (trackid, score) in sorted_scores:
(m, score, actual_match) = tracks[trackid]
print trackid + " (" + str(int(score)) + ", " + str(actual_match) +") - " + m["artist"] + " - " + m["title"]
else:
print "response from fp flat was None -- decoded code was " + str(decoded)
os.remove(new_file)
def reklamaExists(filename,c):
decoded = fp.decode_code_string(c[0]["code"])
result = fp.best_match_for_query(decoded)
if result.TRID:
logfile.write(filename+" is already in the database\n")
return True
else:
return False
def melodyExists(filename, c):
decoded = fp.decode_code_string(c[0]["code"])
result = fp.best_match_for_query(decoded)
if result.TRID:
logfile.write(filename + " is already in the database\n")
return True
else:
return False
def melodyExists(filename,ci,mid):
decoded = fp.decode_code_string(c[0]["code"])
result = fp.best_match_for_query(decoded)
if result.TRID:
updateUploadedMelodyError("Melody already exists in Database",mid,1,0,result.TRID)
return True
else:
return False
def _query_echoprint(self, fingerprint):
decoded = fp.decode_code_string(fingerprint)
result = fp.best_match_for_query(decoded)
if result.TRID:
metadata = result.metadata
else:
raise Exception("No match. This track may not be in the database yet.")
return metadata
def GET(self):
stuff = web.input(fp_code="")
response = fp.best_match_for_query(stuff.fp_code)
track_info = {key: value for key, value in response.metadata.items()
if key != "import_date"}
if "track_id" in track_info.keys():
track_info["track_id"] = track_info["track_id"].split("-")[0]
return json.dumps({"ok":True, "query":stuff.fp_code, "message":response.message(), "match":response.match(), "score":response.score, \
"qtime":response.qtime, "track_id":response.TRID, "total_time":response.total_time, "track_info":track_info})
def lookup(self, log):
#print id,'code list size is ' + str(len(code)) + '!'
code = log['code']
try:
time1 = time.time()
result = fp.best_match_for_query(code)
time2 = time.time()
print 'query time is %0.2f'%(time2- time1) + 's'
if result.TRID:
songName = result.metadata.get("track")
artistName = result.metadata.get("artist")
print "ID: %s" % (result.TRID)
print "Artist: %s" % (artistName)
print "Song: %s" % (songName)
#insert data to song db
query = self.db.query(models.Song)
first = query.filter(models.Song.name == songName, models.Song.artist == artistName).first()
if first is None:
print 'Not in db. Will insert.'
song = models.Song(name = songName, artist = artistName)
self.db.add(song)
self.db.commit()
songId = song.id
else:
songid = first.id
# songId = query.filter(models.Song.name == songName, models.Song.artist == artistName).first().id
listenlog_times = models.Listenlog_times(stationId = log['stationId'], time= log['time'], fingerprint = log['code'], song = songid)
self.db.add(listenlog_times)
self.db.commit()
#check_insert_listenonce
if check_insert_listenonce(3, songid, listenlog_times.stationId):
listenlog_once = models.Listenlog_once(stationId = log['stationId'], time= log['time'], fingerprint = log['code'], song = songid)
self.db.add(listenlog_once)
self.db.commit()
#insert artist info to artist db
query = self.db.query(models.Artist)
first = query.filter(models.Artist.name == artistName).first()
if first is None:
artist = models.Artist(name = artistName)
self.db.add(artist)
self.db.commit()
first = artist
# first = query.filter(models.Artist.name == artistName).first()
if first.photo is None:
ArtistPhotoScanner.getPhoto(self.db, artistName, first.id)
self.db.commit()
else:
print "No match."
listenlog_null = models.Listenlog_null(stationId = log['stationId'], time= log['time'], fingerprint = log['code'])
self.db.add(listenlog_null)
self.db.commit()
except KeyError:
print 'response error'
def reprocess(fid, decoded, date_played, time_played, radio, radio_id):
result = fp.best_match_for_query(decoded)
if result.TRID and result.TRID not in ignored_songs:
logfile.write("Melody is recognized: " + result.TRID + ",")
#Melody is recognized
track_id = result.TRID
current = convertTimeToMinutes(time_played)
global last_time, last_radio, recognized, last_date
conn = MySQLdb.connect(host="localhost",
user="******",
passwd="ulut123",
db="pymusic",
charset='utf8')
#Insert only tracks which differ more than 5 minutes, it is do to prevent repeated insert from different parts of the same track
if (last_radio is None) or (last_date is None) or (
last_date != date_played) or (last_radio != radio_id) or (
last_time == 0) or (current - last_time > 5):
last_time = current
last_radio = radio_id
last_date = date_played
recognized = recognized + 1
db = conn.cursor()
logfile.write("Inserting track_id: " + track_id +
" for fingerprint " + str(fid) + '\n')
db.execute(
"""INSERT INTO played_melody(track_id,radio,date_played,time_played,radio_id) VALUES (%s,%s,%s,%s,%s)""",
(track_id, radio, date_played, time_played, radio_id))
conn.commit()
db.close()
db = conn.cursor()
logfile.write("Updating fingerprint " + str(fid) + " as " +
track_id + '\n')
db.execute(
"""update fingerprint set status = 'Y', track_id = %s, time_identified=%s where id = %s""",
(track_id, getNowDateTime(), fid))
conn.commit()
db.close()
else:
logfile.write(
"Track is already recognized from previous fingerprint " +
radio + str(date_played) + ":" + str(time_played) + '\n')
db = conn.cursor()
logfile.write("Updating fingerprint " + str(fid) + " as Y" + '\n')
db.execute(
"""update fingerprint set status = 'Y', time_identified=%s where id = %s""",
(getNowDateTime(), fid))
conn.commit()
db.close()
conn.close()
return 0
def main():
if not len(sys.argv)==4:
print "usage: python little_eval.py [database_list | disk] query_list [limit]"
sys.exit()
fp_codes = []
limit = int(sys.argv[3])
if sys.argv[1] == "disk":
fp.local_load("disk.pkl")
else:
database_list = open(sys.argv[1]).read().split("\n")[0:limit]
for line in database_list:
(track_id, file) = line.split(" ### ")
print track_id
# TODO - use threaded codegen
j = codegen(file, start=-1, duration=-1)
if len(j):
code_str = fp.decode_code_string(j[0]["code"])
meta = j[0]["metadata"]
l = meta["duration"] * 1000
a = meta["artist"]
r = meta["release"]
t = meta["title"]
v = meta["version"]
fp_codes.append({"track_id": track_id, "fp": code_str, "length": str(l), "codever": str(round(v, 2)), "artist": a, "release": r, "track": t})
fp.ingest(fp_codes, local=True)
fp.local_save("disk.pkl")
counter = 0
actual_win = 0
original_win = 0
bm_win = 0
query_list = open(sys.argv[2]).read().split("\n")[0:limit]
for line in query_list:
(track_id, file) = line.split(" ### ")
print track_id
j = codegen(munge(file))
if len(j):
counter+=1
response = fp.query_fp(fp.decode_code_string(j[0]["code"]), rows=30, local=True, get_data=True)
(winner_actual, winner_original) = get_winners(fp.decode_code_string(j[0]["code"]), response, elbow=8)
winner_actual = winner_actual.split("-")[0]
winner_original = winner_original.split("-")[0]
response = fp.best_match_for_query(j[0]["code"], local=True)
if(response.TRID == track_id):
bm_win+=1
if(winner_actual == track_id):
actual_win+=1
if(winner_original == track_id):
original_win+=1
print "%d / %d actual (%2.2f%%) %d / %d original (%2.2f%%) %d / %d bm (%2.2f%%)" % (actual_win, counter, (float(actual_win)/float(counter))*100.0, \
original_win, counter, (float(original_win)/float(counter))*100.0, \
bm_win, counter, (float(bm_win)/float(counter))*100.0)
def GET(self):
stuff = web.input(fp_code="")
response = fp.best_match_for_query(stuff.fp_code)
track_info = {
key: value
for key, value in response.metadata.items() if key != "import_date"
}
if "track_id" in track_info.keys():
track_info["track_id"] = track_info["track_id"].split("-")[0]
return json.dumps({"ok":True, "query":stuff.fp_code, "message":response.message(), "match":response.match(), "score":response.score, \
"qtime":response.qtime, "track_id":response.TRID, "total_time":response.total_time, "track_info":track_info})
def GET(self):
stuff = web.input(fp_code="")
response = fp.best_match_for_query(stuff.fp_code)
if response.TRID:
artist = response.metadata["artist"]
title = response.metadata["track"]
else:
artist = ""
title = ""
return json.dumps({"ok":True, "query":stuff.fp_code, "message":response.message(), "match":response.match(), "score":response.score, \
"qtime":response.qtime, "track_id":response.TRID, "total_time":response.total_time, "artist":artist,
"title": title})
def GET(self):
data = web.data()
json_data = json.loads(data)
print("Query is: '%s'" % json_data)
response = fp.best_match_for_query(json_data['code'], elbow=10, local=False)
if 'youtube' in response.metadata:
return json.dumps({"ok":True, "query":json_data['code'], "message":response.message(), "match":response.match(), "score":response.score, \
"qtime":response.qtime, "track_id":response.TRID, "total_time":response.total_time, "youtube": response.metadata['youtube'], \
"characters": [eval(val) for val in response.metadata['characters']], "track":response.metadata['track'], "artist":response.metadata['artist']})
else:
return json.dumps({"ok":True, "query":json_data['code'], "message":response.message(), "match":response.match(), "score":response.score, \
"qtime":response.qtime, "track_id":response.TRID, "total_time":response.total_time, "youtube": "", \
"characters":[], "track":"", "artist":""})
def lookup(file):
codes = codegen(file)
if len(codes) and "code" in codes[0]:
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
print "Got result:", result
if result.TRID:
print "ID: %s" % (result.TRID)
print "Artist: %s" % (result.metadata.get("artist"))
print "Song: %s" % (result.metadata.get("track"))
else:
print "No match. This track may not be in the database yet."
else:
print "Couldn't decode", file
def lookup(file):
codes = codegen(file)
if len(codes) and "code" in codes[0]:
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
print "Got result:", result
if result.TRID:
print "ID: %s" % (result.TRID)
print "Artist: %s" % (result.metadata.get("artist"))
print "Song: %s" % (result.metadata.get("track"))
else:
print "No match. This track may not be in the database yet."
else:
print "Couldn't decode", file
def GET(self):
stuff = web.input(fp_code="")
response = fp.best_match_for_query(stuff.fp_code)
return json.dumps(
{
"ok": True,
"query": stuff.fp_code,
"message": response.message(),
"match": response.match(),
"score": response.score,
"qtime": response.qtime,
"track_id": response.TRID,
"total_time": response.total_time,
}
)
def generate_fingerprint_from_list(results, file_list):
# TODO: os.system is thread safe??
# TODO: How to test this?
codes_file = '/tmp/allcodes_%s.json' % (random.randint(1, 10000))
command = '/home/vagrant/echoprint-codegen/echoprint-codegen -s 10 30 < %s > %s' % (file_list, codes_file)
os.system(command)
# Create the Track models
with open(codes_file, 'r') as data_file:
data = json.load(data_file)
for fingerprint in data:
# check fp doesn't exist in database
code_string = fingerprint.get('code')
if code_string:
response = fp.best_match_for_query(code_string)
if not response.match():
label = [v for v in results if v[1] == fingerprint['metadata']['filename']][0][0]
youtube_code = fingerprint['metadata']['filename'].replace('.mp3', '').replace('/tmp/', '')
year = label.split('-')[0].strip()
release = label.split('-')[1].strip()
artist = label.split('-')[2].strip()
title = label.split('-')[3].strip()
fingerprint['metadata']['artist'] = artist
fingerprint['metadata']['title'] = title
# Track creation
Track.sync()
track = Track(band=artist, release=release,
name=title,
year=year,
youtube_code=youtube_code)
track.save()
# Remove all - (due to limitation in fingerprint-server track_id match)
fingerprint['metadata']['track_id'] = track.echoprint_id
else:
# remove duplicate element
data.remove(fingerprint)
print "This file is duplicated"
# Overwrite with artist and title
with open(codes_file, 'w') as data_file:
data_file.write(json.dumps(data))
# Fastingest invoke => post all into echo-fingerprint
codes, _ = parse_json_dump(codes_file)
fp.ingest(codes)
FileHandler.delete_file(codes_file)
return True
def process_file(filename):
codes = codegen(filename)
if codes is None:
return -2
if len(codes) == 0:
logfile.write(getNowDateTime() + ":Codegen returned empty list\n")
return -3
if "code" not in codes[0]:
logfile.write(getNowDateTime() + ":No code is returned by codegen\n")
return -4
track_id = None
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
conn = MySQLdb.connect(host="localhost",
user="******",
passwd="ulut123",
db="pymusic",
charset='utf8')
last_tracks.append(result.TRID)
global last_unknown, last_known
if result.TRID and result.TRID not in ignored_songs:
#Melody is recognized
track_id = result.TRID
#Insert tracks only once
if ((last_known == 0) or
(last_known != track_id)) and moreThanMatchesInLastTracks(
track_id, 1, radio):
last_known = track_id
try:
db = conn.cursor()
db.execute(
"""INSERT INTO played_melody(track_id,radio,date_played,time_played,radio_id) VALUES (%s,%s,%s,%s,%s)""",
(track_id, radio, getNowDate(), getNowTime(), radio_id))
conn.commit()
db.close()
except db.Error, e:
logfile.write(getNowDateTime())
logfile.write(":Error %d: %s\n" % (e.args[0], e.args[1]))
conn.rollback()
raise
def process_file(filename):
codes = codegen(filename)
if codes is None:
return -2
if len(codes) == 0:
logfile.write(getNowDateTime() + ":Codegen returned empty list\n")
return -3
if "code" not in codes[0]:
logfile.write(getNowDateTime() + ":No code is returned by codegen\n")
return -4
db = conn.cursor()
track_id = None
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
if result.TRID and result.TRID not in ignored_songs:
#Melody is recognized
track_id = result.TRID
global last
#Insert tracks only once
if (last == 0) or (last != track_id):
last = track_id
try:
logfile.write(getNowDateTime() + ":Inserting track_id " +
track_id + '\n')
db.execute(
"""INSERT INTO played_melody(track_id,radio,date_played,time_played,radio_id) VALUES (%s,%s,%s,%s,%s)""",
(track_id, radio, getNowDate(), getNowTime(), radio_id))
conn.commit()
except db.Error, e:
logfile.write(getNowDateTime())
logfile.write(":Error %d: %s\n" % (e.args[0], e.args[1]))
conn.rollback()
else:
logfile.write(getNowDateTime() + ":Track " + track_id +
" is already recognized\n")
db.close()
return 0
def lookup(self, code, id):
#print id,'code list size is ' + str(len(code)) + '!'
try:
time1 = time.time()
result = fp.best_match_for_query(code)
time2 = time.time()
print 'query time is %0.2f'%(time2- time1) + 's'
if result.TRID:
songName = result.metadata.get("track")
artistName = result.metadata.get("artist")
print "ID: %s" % (result.TRID)
print "Artist: %s" % (artistName)
print "Song: %s" % (songName)
#insert data to song db
query = self.db.query(models.Song)
first = query.filter(models.Song.name == songName, models.Song.artist == artistName).first()
if first is None:
print 'Not in db. Will insert.'
song = models.Song(name = songName, artist = artistName)
self.db.add(song)
self.db.commit()
songId = query.filter(models.Song.name == songName, models.Song.artist == artistName).first().id
#update listen log db data
query = self.db.query(models.ListenLog)
query.filter(models.ListenLog.id == id).update({models.ListenLog.song: songId})
self.db.commit()
#insert artist info to artist db
query = self.db.query(models.Artist)
first = query.filter(models.Artist.name == artistName).first()
if first is None:
artist = models.Artist(name = artistName)
self.db.add(artist)
self.db.commit()
first = query.filter(models.Artist.name == artistName).first()
if first.photo is None:
ArtistPhotoScanner.getPhoto(self.db, artistName, first.id)
self.db.commit()
else:
print "No match."
except KeyError:
print 'response error'
def match(fileName, code):
response = fp.best_match_for_query(code)
track_info = {key: value for key, value in response.metadata.items() if key != "import_date"}
if "track_id" in track_info.keys():
track_info["track_id"] = track_info["track_id"].split("-")[0]
print json.dumps(
{
"ok": True,
"query": code,
"message": response.message(),
"match": response.match(),
"score": response.score,
"qtime": response.qtime,
"track_id": response.TRID,
"total_time": response.total_time,
"track_info": track_info,
"file_name": fileName,
}
)
sys.stdout.flush()
def process_file(filename,length):
codes = codegen(filename,length)
if codes is None:
return -2
if len(codes) == 0:
logfile.write(getNowDateTime()+":Codegen returned empty list\n")
return -3
if "code" not in codes[0]:
logfile.write(getNowDateTime()+":No code is returned by codegen\n")
return -4
track_id = None
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
conn = MySQLdb.connect(host= "localhost",user="******", passwd="ulut123", db="reklama",charset='utf8')
if result.TRID:
#Melody is recognized
track_id = result.TRID
last_identified_tracks.append((track_id,getNowTime()))
global last_track,last_time
#Insert tracks only once
if (last_track == None or moreThan2MinutesDifference(getNowTime(),last_time) or last_track != track_id) and trackIdentified2TimesInLast2Minutes(track_id):
last_track = track_id
last_time = getNowTime()
try:
db = conn.cursor()
db.execute("""INSERT INTO played_reklama(track_id,radio,date_played,time_played,radio_id,length,filename) VALUES (%s,%s,%s,%s,%s,%s,%s)""",(track_id,radio,getNowDate(),getNowTime(),radio_id,length,filename))
conn.commit()
db.close()
except db.Error, e:
logfile.write(getNowDateTime())
logfile.write(":Error %d: %s\n" % (e.args[0],e.args[1]))
conn.rollback()
raise
conn.close()
return 0
def lookup(log):
code = log['code']
try:
t = time.time()
match_result = fp.best_match_for_query(code)
print "match cost time: ", time.time()-t
if match_result.TRID:
print "Match"
song = match_result.metadata.get("track")
artist = match_result.metadata.get("artist")
print "ID: %s" % match_result.TRID
print "Artist: %s" % artist
print "Song: %s" % song
log["song"] = song
log["artist"] = artist
log['table'] = 1
else:
print "No match"
log["table"] = 0
except KeyError:
print "response error"
log["table"] = 0
return log
def process_file(filename,radio):
now_time = time.strftime('%H:%M:%S')
now_date = time.strftime('%Y-%m-%d')
db = conn.cursor()
codes = codegen(filename)
track_id = None
if len(codes)>0 and "code" in codes[0]:
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
if result.TRID:
track_id = result.TRID
else:
#Insert melody to unknown fingerprints table with status 'N'
logfile.write("No match found for the file, inserting unknown melody to fingerprint table")
db.execute("""INSERT INTO fingerprint(fp,radio,date_played,time_played,time_identified,status) VALUES (%s,%s,%s,%s,%s,%s)""",(decoded,radio,now_date,now_time,None,'N'))
conn.commit()
db.close()
return -1
else:
logfile.write("Couldn't decode "+file+'\n')
db.close()
return -2
global last
#Insert tracks only once
if (last == 0) or (last != track_id):
last=track_id
try:
logfile.write("Inserting track_id: " + track_id + " for file: " + filename+'\n')
db.execute("""INSERT INTO played_melody(track_id,radio,date_played,time_played) VALUES (%s,%s,%s,%s)""",(track_id,radio,now_date,now_time))
conn.commit()
except db.Error, e:
logfile.write("Error %d: %s\n" % (e.args[0],e.args[1]))
conn.rollback()
def GET(self):
stuff = web.input(fp_code="")
response = fp.best_match_for_query(stuff.fp_code)
return json.dumps({"ok":True, "query":stuff.fp_code, "message":response.message(), "match":response.match(), "score":response.score, \
"qtime":response.qtime, "track_id":response.TRID, "total_time":response.total_time})
S = '<sep>'
for dir in sys.argv[1:]:
collect_mp3s(mp3s, dir)
print 'processing', len(mp3s), 'files'
dups = open("dedup.dat", "a")
for count, mp3 in enumerate(mp3s):
if mp3 in done:
print ' skipping', mp3
continue
print " ", len(done), count, mp3
results = codegen(mp3)
if not results:
print "can't process", mp3, "skipping"
continue
trid, raw_code, ingest_data = results
response = fp.best_match_for_query(raw_code)
if response.match():
print >>dups, 'duplicate', S, mp3, S, 'original', S, get_file(response.TRID)
print
print 'duplicate', mp3
print ' ', get_file(response.TRID)
print
else:
fp.ingest([ingest_data], do_commit=True)
done[mp3] = trid
dups.close()
def main():
if not len(sys.argv) == 4:
print "usage: python little_eval.py [database_list | disk] query_list [limit]"
sys.exit()
fp_codes = []
limit = int(sys.argv[3])
if sys.argv[1] == "disk":
fp.local_load("disk.pkl")
else:
database_list = open(sys.argv[1]).read().split("\n")[0:limit]
for line in database_list:
(track_id, file) = line.split(" ### ")
print track_id
# TODO - use threaded codegen
j = codegen(file, start=-1, duration=-1)
if len(j):
code_str = fp.decode_code_string(j[0]["code"])
meta = j[0]["metadata"]
l = meta["duration"] * 1000
a = meta["artist"]
r = meta["release"]
t = meta["title"]
v = meta["version"]
fp_codes.append({
"track_id": track_id,
"fp": code_str,
"length": str(l),
"codever": str(round(v, 2)),
"artist": a,
"release": r,
"track": t
})
fp.ingest(fp_codes, local=True)
fp.local_save("disk.pkl")
counter = 0
actual_win = 0
original_win = 0
bm_win = 0
query_list = open(sys.argv[2]).read().split("\n")[0:limit]
for line in query_list:
(track_id, file) = line.split(" ### ")
print track_id
j = codegen(munge(file))
if len(j):
counter += 1
response = fp.query_fp(fp.decode_code_string(j[0]["code"]),
rows=30,
local=True,
get_data=True)
(winner_actual, winner_original) = get_winners(
fp.decode_code_string(j[0]["code"]), response, elbow=8)
winner_actual = winner_actual.split("-")[0]
winner_original = winner_original.split("-")[0]
response = fp.best_match_for_query(j[0]["code"], local=True)
if (response.TRID == track_id):
bm_win += 1
if (winner_actual == track_id):
actual_win += 1
if (winner_original == track_id):
original_win += 1
print "%d / %d actual (%2.2f%%) %d / %d original (%2.2f%%) %d / %d bm (%2.2f%%)" % (actual_win, counter, (float(actual_win)/float(counter))*100.0, \
original_win, counter, (float(original_win)/float(counter))*100.0, \
bm_win, counter, (float(bm_win)/float(counter))*100.0)
mp3s = []
S = '<sep>'
for dir in sys.argv[1:]:
collect_mp3s(mp3s, dir)
print 'processing', len(mp3s), 'files'
dups = open("dedup.dat", "a")
for count, mp3 in enumerate(mp3s):
if mp3 in done:
print ' skipping', mp3
continue
print " ", len(done), count, mp3
results = codegen(mp3, start=0, duration=60)
if not results:
print "can't process", mp3, "skipping"
continue
trid, raw_code, ingest_data = results
response = fp.best_match_for_query(raw_code)
if response.match():
print >> dups, 'duplicate', S, mp3, S, 'original', S, get_file(
response.TRID)
print
print 'duplicate', mp3
print ' ', get_file(response.TRID)
print
else:
fp.ingest([ingest_data], do_commit=True)
done[mp3] = trid
dups.close()
def test_file(filename,
local=False,
expect_match=True,
original_TRID=None,
remove_file=True):
"""
Test a single file. This will return a code like tn, tp, fp, err, etc
"""
matching_TRID = None
if filename is None:
return "err-munge" # most likely a munge error (file too short, unreadable, etc)
try:
# don't pass start and duration to codegen, assume the munging takes care of codelength
if not local:
query_obj = song.util.codegen(filename, start=-1, duration=-1)
s = fp.best_match_for_query(query_obj[0]["code"])
if s.TRID is not None:
matching_TRID = s.TRID
else:
query_obj = song.util.codegen(filename, start=-1, duration=-1)
s = fp.best_match_for_query(query_obj[0]["code"], local=local)
if s.TRID is not None:
matching_TRID = s.TRID
except IOError:
print "TIMEOUT from API server"
return "err-api"
except TypeError: # codegen returned none
return "err-codegen"
if remove_file:
if os.path.exists(filename):
os.remove(filename)
else:
return "err-munge"
# if is not None there was a response
if s is not None:
# there was a match
if len(s) > 0:
# if we're expecting a match, check that it's the right one
if expect_match:
if original_TRID == matching_TRID:
# we expected a match, it's the right one. TP
return "tp"
else:
# we expected a match but it's the wrong one. FP-a
return "fp-a"
else:
# we did not expect a match. FP-b
return "fp-b"
else:
# there was no match from the API
if expect_match:
# we expected a match. FN
return "fn"
else:
# we did not expect a match. TN
return "tn"
else:
# s is None, that means API error-- almost definitely codegen returned nothing.
return "err-codegen"
