def main():
if not len(sys.argv)==4:
print "usage: python little_eval.py [database_list | disk] query_list [limit]"
sys.exit()
fp_codes = []
limit = int(sys.argv[3])
if sys.argv[1] == "disk":
fp.local_load("disk.pkl")
else:
database_list = open(sys.argv[1]).read().split("\n")[0:limit]
for line in database_list:
(track_id, file) = line.split(" ### ")
print track_id
# TODO - use threaded codegen
j = codegen(file, start=-1, duration=-1)
if len(j):
code_str = fp.decode_code_string(j[0]["code"])
meta = j[0]["metadata"]
l = meta["duration"] * 1000
a = meta["artist"]
r = meta["release"]
t = meta["title"]
v = meta["version"]
fp_codes.append({"track_id": track_id, "fp": code_str, "length": str(l), "codever": str(round(v, 2)), "artist": a, "release": r, "track": t})
fp.ingest(fp_codes, local=True)
fp.local_save("disk.pkl")
counter = 0
actual_win = 0
original_win = 0
bm_win = 0
query_list = open(sys.argv[2]).read().split("\n")[0:limit]
for line in query_list:
(track_id, file) = line.split(" ### ")
print track_id
j = codegen(munge(file))
if len(j):
counter+=1
response = fp.query_fp(fp.decode_code_string(j[0]["code"]), rows=30, local=True, get_data=True)
(winner_actual, winner_original) = get_winners(fp.decode_code_string(j[0]["code"]), response, elbow=8)
winner_actual = winner_actual.split("-")[0]
winner_original = winner_original.split("-")[0]
response = fp.best_match_for_query(j[0]["code"], local=True)
if(response.TRID == track_id):
bm_win+=1
if(winner_actual == track_id):
actual_win+=1
if(winner_original == track_id):
original_win+=1
print "%d / %d actual (%2.2f%%) %d / %d original (%2.2f%%) %d / %d bm (%2.2f%%)" % (actual_win, counter, (float(actual_win)/float(counter))*100.0, \
original_win, counter, (float(original_win)/float(counter))*100.0, \
bm_win, counter, (float(bm_win)/float(counter))*100.0)
def test_single(filename, local=False, **munge_kwargs):
"""
Perform a test on a single file. Prints more diagnostic information than usual.
"""
(new_file, what) = munge(filename, **munge_kwargs)
query_obj = song.util.codegen(new_file, start=-1, duration=-1)
s = fp.best_match_for_query(query_obj[0]["code"], local=local)
if s.TRID is not None:
if local:
metad = _local_bigeval[s.TRID]
else:
metad = fp.metadata_for_track_id(s.TRID)
metad["title"] = metad["track"]
song_metadata = {
"artist": metad.get("artist", ""),
"release": metad.get("release", ""),
"title": metad.get("title", "")
}
print str(song_metadata)
else:
print "No match"
decoded = fp.decode_code_string(query_obj[0]["code"])
print str(len(decoded.split(" ")) / 2) + " codes in original"
response = fp.query_fp(decoded, local=local, rows=15)
if response is not None:
print "From FP flat:"
tracks = {}
scores = {}
for r in response.results:
trid = r["track_id"].split("-")[0]
if local:
metad = _local_bigeval[trid]
else:
metad = fp.metadata_for_track_id(trid)
metad["title"] = metad["track"]
m = {
"artist": metad.get("artist", ""),
"release": metad.get("release", ""),
"title": metad.get("title", "")
}
if m is not None:
actual_match = fp.actual_matches(
decoded, fp.fp_code_for_track_id(r["track_id"],
local=local))
tracks[r["track_id"]] = (m, r["score"], actual_match)
scores[r["track_id"]] = actual_match
else:
print "problem getting metadata for " + r["track_id"]
sorted_scores = sorted(scores.iteritems(),
key=lambda (k, v): (v, k),
reverse=True)
for (trackid, score) in sorted_scores:
(m, score, actual_match) = tracks[trackid]
print trackid + " (" + str(int(score)) + ", " + str(
actual_match) + ") - " + m["artist"] + " - " + m["title"]
else:
print "response from fp flat was None -- decoded code was " + str(
decoded)
os.remove(new_file)
def query(request):
mp3 = request.FILES['mp3']
# Convert music -> fingerprint code
fp_code = generate_fingerprint(mp3)
if fp_code is dict:
return HttpResponse(fp_code['error'], status=400)
# First see if this is a compressed code
if re.match('[A-Za-z\/\+\_\-]', fp_code) is not None:
code_string = fp.decode_code_string(fp_code)
if code_string is None:
result = json.dumps({"error": "cannot decode code string %s" % fp_code})
return HttpResponse(result, status=400)
else:
code_string = fp_code
response = fp.best_match_for_query(code_string)
metadata = response.metadata
if metadata:
metadata.pop('import_date')
data = json.dumps({"ok": True, "message": response.message(), "match": response.match(),
"score": response.score, \
"qtime": response.qtime, "track_id": response.TRID,
"total_time": response.total_time,
"metadata": metadata})
return HttpResponse(data, status=200)
def ingest(params, mp3):
fp_code = generate_fingerprint(mp3)
if fp_code is dict:
return False
if params.get('track_id', "default") == "default":
track_id = fp.new_track_id()
else:
track_id = params['track_id']
if not params.get('length', None):
return False
# First see if this is a compressed code
if re.match('[A-Za-z\/\+\_\-]', fp_code) is not None:
code_string = fp.decode_code_string(fp_code)
if code_string is None:
result = json.dumps({"track_id": track_id, "ok": False,
"error": "cannot decode code string %s" % fp_code})
return False, result
else:
code_string = fp_code
data = {"track_id": track_id,
"fp": code_string,
"length": params['length'],
"codever": params['codever']}
if params.get('artist'):
data["artist"] = params.get('artist')
if params.get('release'):
data["release"] = params.get('release')
if params.get('track'):
data["track"] = params.get('track')
fp.ingest(data, do_commit=True, local=False)
def parse_json_dump(jfile):
codes = json.load(open(jfile))
bigeval = {}
fullcodes = []
for c in codes:
try:
if "code" not in c:
continue
code = c["code"]
m = c["metadata"]
if "track_id" in m:
trid = m["track_id"].encode("utf-8")
else:
trid = fp.new_track_id()
length = m["duration"]
version = m["version"]
artist = m.get("artist", None)
title = m.get("title", None)
release = m.get("release", None)
decoded = fp.decode_code_string(code)
bigeval[trid] = m
data = {"track_id": trid,
"fp": decoded,
"length": length,
"codever": "%.2f" % version
}
if artist: data["artist"] = artist
if release: data["release"] = release
if title: data["track"] = title
fullcodes.append(data)
except Exception, e:
print 'Exception:', e
def lookup(FILE):
codes = codegen(FILE)
if len(codes) and "code" in codes[0]:
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
print "Got result:", result
if result.TRID:
#I want to print thr current time and the name of music together
ID = "ID: %s" % (result.TRID)
myFile.write(ID+str(CurrentTime))#+"\n"+"Artist: %s" % (result.metadata.get("artist"))+"\n"+"Song: %s" % (result.metadata.get("track"+"\n"))+"\n")
myFile.write("Artist: %s" % (result.metadata.get("artist")))
myFile.write("Song: %s" % (result.metadata.get("track"))+"\n")
#print ID+str(CurrentTime)
#print "Artist: %s" % (result.metadata.get("artist"))
#print "Song: %s" % (result.metadata.get("track"+"\n"))
else:
#print "No match. This track may not be in the database yet."
Unmatch = "No match. This track may not be in the database yet."
myFile.write(Unmatch+str(CurrentTime)+"\n")
else:
Undecode = "Couldn't decode" + str(FILE)
myFile.write(Undecode)
def parse_json(c):
code = c["code"]
m = c["metadata"]
if "track_id" in m:
trid = m["track_id"].encode("utf-8")
else:
trid = fp.new_track_id()
length = m["duration"]
version = m["version"]
artist = m.get("artist", None)
title = m.get("title", None)
release = m.get("release", None)
decoded = fp.decode_code_string(code)
data = {"track_id": trid,
"fp": decoded,
"length": length,
"codever": "%.2f" % version
}
if artist: data["artist"] = artist
if release: data["release"] = release
if title: data["track"] = title
return data
def parse_json(c):
code = c["code"]
m = c["metadata"]
if "track_id" in m:
trid = m["track_id"].encode("utf-8")
else:
trid = fp.new_track_id()
length = m["duration"]
version = m["version"]
artist = m.get("artist", None)
title = m.get("title", None)
release = m.get("release", None)
decoded = fp.decode_code_string(code)
data = {
"track_id": trid,
"fp": decoded,
"length": length,
"codever": "%.2f" % version
}
if artist: data["artist"] = artist
if release: data["release"] = release
if title: data["track"] = title
return data
def parse_json_dump(jfile):
codes = json.load(open(jfile))
bigeval = {}
fullcodes = []
for c in codes:
if "code" not in c:
continue
code = c["code"]
m = c["metadata"]
if "track_id" in m:
trid = m["track_id"].encode("utf-8")
else:
trid = fp.new_track_id()
length = m["duration"]
version = m["version"]
artist = m.get("artist", None)
title = m.get("title", None)
release = m.get("release", None)
decoded = fp.decode_code_string(code)
bigeval[trid] = m
data = {
"track_id": trid,
"fp": decoded,
"length": length,
"codever": "%.2f" % version
}
if artist: data["artist"] = artist
if release: data["release"] = release
if title: data["track"] = title
fullcodes.append(data)
return (fullcodes, bigeval)
def lookup(FILE):
codes = codegen(FILE)
if len(codes) and "code" in codes[0]:
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
print "Got result:", result
if result.TRID:
#I want to print thr current time and the name of music together
ID = "ID: %s" % (result.TRID)
myFile.write(
ID + str(CurrentTime)
) #+"\n"+"Artist: %s" % (result.metadata.get("artist"))+"\n"+"Song: %s" % (result.metadata.get("track"+"\n"))+"\n")
myFile.write("Artist: %s" % (result.metadata.get("artist")))
myFile.write("Song: %s" % (result.metadata.get("track")) +
"\n")
#print ID+str(CurrentTime)
#print "Artist: %s" % (result.metadata.get("artist"))
#print "Song: %s" % (result.metadata.get("track"+"\n"))
else:
#print "No match. This track may not be in the database yet."
Unmatch = "No match. This track may not be in the database yet."
myFile.write(Unmatch + str(CurrentTime) + "\n")
else:
Undecode = "Couldn't decode" + str(FILE)
myFile.write(Undecode)
def parse_json_dump(jfile):
codes = json.load(open(jfile))
fullcodes = []
trids = []
for c in codes:
if "code" not in c:
continue
code = c["code"]
m = c["metadata"]
trid = find_ad_id(m["filename"])
length = m["duration"]
version = m["version"]
artist = m.get("artist",None)
title = m.get("title",None)
release = m.get("release",None)
decoded = fp.decode_code_string(code)
data = {"track_id": trid,
"fp": decoded,
"length": length,
"codever": "%.2f" % version
}
fullcodes.append(data)
trids.append(trid)
return fullcodes,trids
def POST(self):
params = web.input(
track_id="default", fp_code="", artist=None, release=None, track=None, length=None, codever=None
)
if params.track_id == "default":
track_id = fp.new_track_id()
else:
track_id = params.track_id
if params.length is None or params.codever is None:
return web.webapi.BadRequest()
# First see if this is a compressed code
if re.match("[A-Za-z\/\+\_\-]", params.fp_code) is not None:
code_string = fp.decode_code_string(params.fp_code)
if code_string is None:
return json.dumps(
{"track_id": track_id, "ok": False, "error": "cannot decode code string %s" % params.fp_code}
)
else:
code_string = params.fp_code
data = {"track_id": track_id, "fp": code_string, "length": params.length, "codever": params.codever}
if params.artist:
data["artist"] = params.artist
if params.release:
data["release"] = params.release
if params.track:
data["track"] = params.track
fp.ingest(data, do_commit=True, local=False)
return json.dumps({"track_id": track_id, "status": "ok"})
def process_file(filename, t):
codes = codegen(filename)
if codes is None:
return -2
if len(codes) == 0:
return -3
if "code" not in codes[0]:
return -4
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
if result.TRID:
#Melody is recognized
track_id = result.TRID
global last, time
#Insert tracks only once
if (last == 0) or (last != track_id):
last = track_id
mytime = time + datetime.timedelta(seconds=t)
conn = MySQLdb.connect(host="localhost",
user="******",
passwd="ulut123",
db="pymusic",
charset='utf8')
db = conn.cursor()
db.execute(
"""INSERT INTO test_played_melody(track_id,radio,date_played,time_played,radio_id) VALUES (%s,%s,%s,%s,%s)""",
(track_id, radio, date, mytime.time(), radio_id))
conn.commit()
db.close()
conn.close()
def melodyExists(filename,ci,mid):
decoded = fp.decode_code_string(c[0]["code"])
result = fp.best_match_for_query(decoded)
if result.TRID:
updateUploadedMelodyError("Melody already exists in Database",mid,1,0,result.TRID)
return True
else:
return False
def melodyExists(filename, c):
decoded = fp.decode_code_string(c[0]["code"])
result = fp.best_match_for_query(decoded)
if result.TRID:
logfile.write(filename + " is already in the database\n")
return True
else:
return False
def reklamaExists(filename,c):
decoded = fp.decode_code_string(c[0]["code"])
result = fp.best_match_for_query(decoded)
if result.TRID:
logfile.write(filename+" is already in the database\n")
return True
else:
return False
def _query_echoprint(self, fingerprint):
decoded = fp.decode_code_string(fingerprint)
result = fp.best_match_for_query(decoded)
if result.TRID:
metadata = result.metadata
else:
raise Exception("No match. This track may not be in the database yet.")
return metadata
def POST(self):
params = web.input(track_id="default",
fp_code="",
artist=None,
release=None,
track=None,
length=None,
codever=None,
source=None,
genre=None,
bitrate=None,
sample_rate=None)
print params
if params.track_id == "default":
track_id = fp.new_track_id()
else:
track_id = params.track_id
if params.length is None or params.codever is None:
return web.webapi.BadRequest()
# First see if this is a compressed code
if re.match('[A-Za-z\/\+\_\-]', params.fp_code) is not None:
code_string = fp.decode_code_string(params.fp_code)
if code_string is None:
return json.dumps({
"track_id":
track_id,
"ok":
False,
"error":
"cannot decode code string %s" % params.fp_code
})
else:
code_string = params.fp_code
data = {
"track_id": track_id,
"fp": code_string,
"length": params.length,
"codever": params.codever
}
if params.artist: data["artist"] = params.artist
if params.release: data["release"] = params.release
if params.track: data["track"] = params.track
if params.source: data["source"] = params.source
if params.genre: data["genre"] = params.genre
if params.bitrate: data["bitrate"] = params.bitrate
if params.sample_rate: data["sample_rate"] = params.sample_rate
fp.ingest(data, do_commit=True, local=False)
return json.dumps({"track_id": track_id, "status": "ok"})
def ingest(request):
# TODO: Add Track object
params = request.POST
mp3 = request.FILES['mp3']
fp_code = generate_fingerprint(mp3)
if fp_code is dict:
return HttpResponse(fp_code['error'], status=400)
if params.get('track_id', "default") == "default":
track_id = fp.new_track_id()
else:
track_id = params['track_id']
if not params.get('length', None):
return HttpResponse("Invalid data", status=400)
# First see if this is a compressed code
if re.match('[A-Za-z\/\+\_\-]', fp_code) is not None:
code_string = fp.decode_code_string(fp_code)
if code_string is None:
result = json.dumps( {"track_id": track_id, "ok": False,
"error": "cannot decode code string %s" % fp_code})
return HttpResponse(result, status=400)
else:
code_string = fp_code
artist = params.get('artist')
release = params.get('release')
title = params.get('track')
year = params.get('year')
youtube_code = params.get('youtube_code')
# Track creation
track = Track(band=artist, release=release,
name=title,
year=year,
youtube_code=youtube_code)
track.save()
# Remove all - (due to limitation in fingerprint-server track_id match)
track_id = track.echoprint_id
data = {"track_id": track_id,
"fp": code_string,
"length": params['length'],
"codever": params['codever']}
if params.get('artist'):
data["artist"] = params.get('artist')
if params.get('release'):
data["release"] = params.get('release')
if params.get('track'):
data["track"] = params.get('track')
fp.ingest(data, do_commit=True, local=False)
data = json.dumps({"track_id": track_id, "fp": fp_code,"status": "ok"})
return HttpResponse(data, status=200)
def lookup(file):
codes = codegen(file)
if len(codes) and "code" in codes[0]:
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
print "Got result:", result
if result.TRID:
print "ID: %s" % (result.TRID)
print "Artist: %s" % (result.metadata.get("artist"))
print "Song: %s" % (result.metadata.get("track"))
else:
print "No match. This track may not be in the database yet."
else:
print "Couldn't decode", file
def lookup(file):
codes = codegen(file)
if len(codes) and "code" in codes[0]:
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
print "Got result:", result
if result.TRID:
print "ID: %s" % (result.TRID)
print "Artist: %s" % (result.metadata.get("artist"))
print "Song: %s" % (result.metadata.get("track"))
else:
print "No match. This track may not be in the database yet."
else:
print "Couldn't decode", file
def POST(self):
params = web.input(track_id="default", fp_code="", artist=None, release=None, track=None, length=None, codever=None)
if params.track_id == "default":
track_id = fp.new_track_id()
else:
track_id = params.track_id
if params.length is None or params.codever is None:
return web.webapi.BadRequest()
# First see if this is a compressed code
if re.match('[A-Za-z\/\+\_\-]', params.fp_code) is not None:
code_string = fp.decode_code_string(params.fp_code)
# if code_string is None:
return json.dumps({"track_id":track_id, "ok":False, "error":"cannot decode code string %s" % params.fp_code})
def test_single(filename, local=False, **munge_kwargs):
"""
Perform a test on a single file. Prints more diagnostic information than usual.
"""
(new_file, what) = munge(filename, **munge_kwargs)
query_obj = song.util.codegen(new_file, start=-1, duration=-1)
if not local:
s = fp.best_match_for_query(query_obj[0]["code"])
if s.TRID is not None:
metad = _local_bigeval[s.TRID]
song_metadata = {"artist": metad["artist"], "release": metad["release"], "title": metad["title"]}
print str(song_metadata)
else:
print "No match"
else:
s = fp.best_match_for_query(query_obj[0]["code"],local=local)
if s.TRID is not None:
metad = _local_bigeval[s.TRID]
song_metadata = {"artist": metad["artist"], "release": metad["release"], "title": metad["title"]}
print str(song_metadata)
else:
print "No match"
decoded = fp.decode_code_string(query_obj[0]["code"])
print str(len(decoded.split(" "))/2) + " codes in original"
response = fp.query_fp(decoded, local=local, rows=15)
if response is not None:
print "From FP flat:"
tracks = {}
scores = {}
for r in response.results:
trid = r["track_id"].split("-")[0]
metad = _local_bigeval[trid]
m = {"artist": metad["artist"], "release": metad["release"], "title": metad["title"]}
if m is not None:
actual_match = fp.actual_matches(decoded, fp.fp_code_for_track_id(r["track_id"], local=local))
tracks[r["track_id"]] = (m, r["score"], actual_match)
scores[r["track_id"]] = actual_match
else:
print "problem getting metadata for " + r["track_id"]
sorted_scores = sorted(scores.iteritems(), key=lambda (k,v): (v,k), reverse=True)
for (trackid, score) in sorted_scores:
(m, score, actual_match) = tracks[trackid]
print trackid + " (" + str(int(score)) + ", " + str(actual_match) +") - " + m["artist"] + " - " + m["title"]
else:
print "response from fp flat was None -- decoded code was " + str(decoded)
os.remove(new_file)
def magic_matches_list(code_string, elbow=10, local=False):
# DEC strings come in as unicode so we have to force them to ASCII
code_string = code_string.encode("utf8")
# First see if this is a compressed code
if re.match('[A-Za-z\/\+\_\-]', code_string) is not None:
code_string = decode_code_string(code_string)
if code_string is None:
return False
# Query the FP flat directly.
response = query_fp(code_string, rows=30, local=local, get_data=True)
if len(response.results) == 0:
return False
# Get the actual score for all responses
original_scores = {}
actual_scores = {}
trackids = [r["track_id"].encode("utf8") for r in response.results]
tcodes = get_tyrant().multi_get(trackids)
# For each result compute the "actual score" (based on the histogram matching)
for (i, r) in enumerate(response.results):
track_id = r["track_id"]
original_scores[track_id] = int(r["score"])
track_code = tcodes[i]
if track_code is None:
# Solr gave us back a track id but that track
# is not in our keystore
continue
actual_scores[track_id] = actual_matches(code_string, track_code, elbow = elbow)
# Sort the actual scores
sorted_actual_scores = sorted(actual_scores.iteritems(), key=lambda (k,v): (v,k), reverse=True)
# Because we split songs up into multiple parts, sometimes the results will have the same track in the
# first few results. Remove these duplicates so that the falloff is (potentially) higher.
final_results = []
for trid, result in sorted_actual_scores:
trid = trid.split("-")[0]
meta = metadata_for_track_id(trid, local=local)
final_results.append(meta)
return final_results
def local_ingest_json(json):
fp_codes = []
#fp.local_load("disk.pkl")
if len(json):
track_id = fp.new_track_id()
code_str = fp.decode_code_string(json[0]["code"])
meta = json[0]["metadata"]
l = meta["duration"] * 1000
a = meta["artist"]
r = meta["release"]
t = meta["title"]
v = meta["version"]
fp_codes.append({"track_id": track_id, "fp": code_str, "length": str(l), "codever": str(round(v, 2)), "artist": a, "release": r, "track": t})
fp.ingest(fp_codes, local=True)
#fp.local_save("disk.pkl")
return track_id
def process_file(filename):
codes = codegen(filename)
if codes is None:
return -2
if len(codes) == 0:
logfile.write(getNowDateTime() + ":Codegen returned empty list\n")
return -3
if "code" not in codes[0]:
logfile.write(getNowDateTime() + ":No code is returned by codegen\n")
return -4
track_id = None
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
conn = MySQLdb.connect(host="localhost",
user="******",
passwd="ulut123",
db="pymusic",
charset='utf8')
last_tracks.append(result.TRID)
global last_unknown, last_known
if result.TRID and result.TRID not in ignored_songs:
#Melody is recognized
track_id = result.TRID
#Insert tracks only once
if ((last_known == 0) or
(last_known != track_id)) and moreThanMatchesInLastTracks(
track_id, 1, radio):
last_known = track_id
try:
db = conn.cursor()
db.execute(
"""INSERT INTO played_melody(track_id,radio,date_played,time_played,radio_id) VALUES (%s,%s,%s,%s,%s)""",
(track_id, radio, getNowDate(), getNowTime(), radio_id))
conn.commit()
db.close()
except db.Error, e:
logfile.write(getNowDateTime())
logfile.write(":Error %d: %s\n" % (e.args[0], e.args[1]))
conn.rollback()
raise
def process_file(filename):
codes = codegen(filename)
if codes is None:
return -2
if len(codes) == 0:
logfile.write(getNowDateTime() + ":Codegen returned empty list\n")
return -3
if "code" not in codes[0]:
logfile.write(getNowDateTime() + ":No code is returned by codegen\n")
return -4
db = conn.cursor()
track_id = None
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
if result.TRID and result.TRID not in ignored_songs:
#Melody is recognized
track_id = result.TRID
global last
#Insert tracks only once
if (last == 0) or (last != track_id):
last = track_id
try:
logfile.write(getNowDateTime() + ":Inserting track_id " +
track_id + '\n')
db.execute(
"""INSERT INTO played_melody(track_id,radio,date_played,time_played,radio_id) VALUES (%s,%s,%s,%s,%s)""",
(track_id, radio, getNowDate(), getNowTime(), radio_id))
conn.commit()
except db.Error, e:
logfile.write(getNowDateTime())
logfile.write(":Error %d: %s\n" % (e.args[0], e.args[1]))
conn.rollback()
else:
logfile.write(getNowDateTime() + ":Track " + track_id +
" is already recognized\n")
db.close()
return 0
def parse_json_dump(self, json_data):
codes = json_data
bigeval = {}
fullcodes = []
track_ids = []
for c in codes:
if "code" not in c:
continue
code = c["code"]
m = c["metadata"]
if "track_id" in m:
trid = m["track_id"].encode("utf-8")
else:
trid = fp.new_track_id()
track_ids.append(trid)
length = m["duration"]
version = m["version"]
artist = m.get("artist", None)
title = m.get("title", None)
release = m.get("release", None)
youtube = m.get("youtube", None)
characters = m.get("characters", None)
decoded = fp.decode_code_string(code)
bigeval[trid] = m
data = {"track_id": trid,
"fp": decoded,
"length": length,
"codever": "%.2f" % version
}
if artist: data["artist"] = artist
if release: data["release"] = release
if title: data["track"] = title
if youtube: data["youtube"] = youtube
if characters: data["characters"] = characters
fullcodes.append(data)
return (fullcodes, bigeval, track_ids)
def process_file(filename,length):
codes = codegen(filename,length)
if codes is None:
return -2
if len(codes) == 0:
logfile.write(getNowDateTime()+":Codegen returned empty list\n")
return -3
if "code" not in codes[0]:
logfile.write(getNowDateTime()+":No code is returned by codegen\n")
return -4
track_id = None
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
conn = MySQLdb.connect(host= "localhost",user="******", passwd="ulut123", db="reklama",charset='utf8')
if result.TRID:
#Melody is recognized
track_id = result.TRID
last_identified_tracks.append((track_id,getNowTime()))
global last_track,last_time
#Insert tracks only once
if (last_track == None or moreThan2MinutesDifference(getNowTime(),last_time) or last_track != track_id) and trackIdentified2TimesInLast2Minutes(track_id):
last_track = track_id
last_time = getNowTime()
try:
db = conn.cursor()
db.execute("""INSERT INTO played_reklama(track_id,radio,date_played,time_played,radio_id,length,filename) VALUES (%s,%s,%s,%s,%s,%s,%s)""",(track_id,radio,getNowDate(),getNowTime(),radio_id,length,filename))
conn.commit()
db.close()
except db.Error, e:
logfile.write(getNowDateTime())
logfile.write(":Error %d: %s\n" % (e.args[0],e.args[1]))
conn.rollback()
raise
conn.close()
return 0
def process_file(filename,radio):
now_time = time.strftime('%H:%M:%S')
now_date = time.strftime('%Y-%m-%d')
db = conn.cursor()
codes = codegen(filename)
track_id = None
if len(codes)>0 and "code" in codes[0]:
decoded = fp.decode_code_string(codes[0]["code"])
result = fp.best_match_for_query(decoded)
if result.TRID:
track_id = result.TRID
else:
#Insert melody to unknown fingerprints table with status 'N'
logfile.write("No match found for the file, inserting unknown melody to fingerprint table")
db.execute("""INSERT INTO fingerprint(fp,radio,date_played,time_played,time_identified,status) VALUES (%s,%s,%s,%s,%s,%s)""",(decoded,radio,now_date,now_time,None,'N'))
conn.commit()
db.close()
return -1
else:
logfile.write("Couldn't decode "+file+'\n')
db.close()
return -2
global last
#Insert tracks only once
if (last == 0) or (last != track_id):
last=track_id
try:
logfile.write("Inserting track_id: " + track_id + " for file: " + filename+'\n')
db.execute("""INSERT INTO played_melody(track_id,radio,date_played,time_played) VALUES (%s,%s,%s,%s)""",(track_id,radio,now_date,now_time))
conn.commit()
except db.Error, e:
logfile.write("Error %d: %s\n" % (e.args[0],e.args[1]))
conn.rollback()
def parse_json_block(json_block):
codes = json.loads(json_block)
code = codes[0]
if not 'code' in code:
return None
raw_code = code["code"]
m = code["metadata"]
if "track_id" in m:
trid = m["track_id"].encode("utf-8")
else:
trid = fp.new_track_id()
length = m["duration"]
version = m["version"]
artist = m.get("artist", None)
title = m.get("title", None)
release = m.get("release", None)
decoded = fp.decode_code_string(raw_code)
ingest_data = {
"track_id": trid,
"fp": decoded,
"length": length,
"codever": "%.2f" % version
}
if artist:
ingest_data["artist"] = artist
if release:
ingest_data["release"] = release
if title:
ingest_data["track"] = title
return trid, raw_code, ingest_data
def parse_json_block(json_block):
codes = json.loads(json_block)
code = codes[0]
if not 'code' in code:
return None
raw_code = code["code"]
m = code["metadata"]
if "track_id" in m:
trid = m["track_id"].encode("utf-8")
else:
trid = fp.new_track_id()
length = m["duration"]
version = m["version"]
artist = m.get("artist", None)
title = m.get("title", None)
release = m.get("release", None)
decoded = fp.decode_code_string(raw_code)
ingest_data = {
"track_id": trid,
"fp": decoded,
"length": length,
"codever": "%.2f" % version
}
if artist:
ingest_data["artist"] = artist
if release:
ingest_data["release"] = release
if title:
ingest_data["track"] = title
return trid, raw_code, ingest_data
def main(argv):
global _local_bigeval, _new_music_files
global _new_queries, _old_queries, _total_queries
single = None
how_many = None
start = 0
duration = 30
bitrate = 128
volume = -1
lowpass = -1
decoder = "mpg123"
encoder = "wav"
local = None
diag = False
channels = 2
downsample = False
decoder = "mpg123"
testfile = os.path.join(os.path.dirname(__file__), 'bigeval.json')
newfile = "new_music"
no_shuffle = False
try:
opts, args = getopt.getopt(argv, "1:c:s:d:D:b:v:l:L:e:B:t:n:pm2h", [
"single=", "count=", "start=", "duration=", "decoder=", "bitrate=",
"volume=", "lowpass="******"encoder=", "print", "mono", "local=",
"test=", "new=", "22kHz", "help", "no-shuffle"
])
except getopt.GetoptError:
usage()
sys.exit(2)
for opt, arg in opts:
if opt in ("-1", "--single"):
single = arg
if opt in ("-c", "--count"):
how_many = int(arg)
if opt in ("-s", "--start"):
start = int(arg)
if opt in ("-d", "--duration"):
duration = int(arg)
if opt in ("-D", "--decoder"):
decoder = arg
if opt in ("-b", "--bitrate"):
bitrate = int(arg)
if opt in ("-v", "--volume"):
volume = int(arg)
if opt in ("-l", "--lowpass"):
lowpass = int(arg)
if opt in ("-e", "--encoder"):
encoder = arg
if opt in ("-L", "--local"):
local = arg
if opt in ("-p", "--print"):
diag = True
if opt in ("-m", "--mono"):
channels = 1
if opt in ("-2", "--22kHz"):
downsample = True
if opt in ("-B", "--binary"):
if not os.path.exists(arg):
print "Binary %s not found. Exiting." % arg
sys.exit(2)
config.CODEGEN_BINARY_OVERRIDE = arg
if opt in ("-n", "--new"):
newfile = arg
if opt in ("-t", "--test"):
testfile = arg
if opt == "--no-shuffle":
no_shuffle = True
if opt in ("-h", "--help"):
usage()
sys.exit(2)
if (single is None) and (how_many is None):
print >> sys.stderr, "Run in single mode (-1) or say how many files to test (-c)"
usage()
sys.exit(2)
if testfile.lower() == "none" and newfile.lower(
) == "none" and single is None:
# If both are none, we can't run
print >> sys.stderr, "Can't run with no datafiles. Skip --test, --new or add -1"
sys.exit(2)
if testfile.lower() == "none":
_local_bigeval = {}
else:
if not os.path.exists(testfile):
print >> sys.stderr, "Cannot find bigeval.json. did you run fastingest with the -b flag?"
sys.exit(1)
_local_bigeval = json.load(open(testfile, 'r'))
if newfile.lower() == "none" or not os.path.exists(newfile):
_new_music_files = []
else:
_new_music_files = open(newfile, 'r').read().split('\n')
_new_queries = float(len(_new_music_files))
_old_queries = float(len(_local_bigeval.keys()))
_total_queries = _new_queries + _old_queries
if local is None:
local = False
else:
# ingest
codes = json.load(open(local, 'r'))
_reversed_bigeval = dict(
(_local_bigeval[k], k) for k in _local_bigeval)
code_dict = {}
tids = {}
for c in codes:
fn = c["metadata"]["filename"]
tid = _reversed_bigeval.get(fn, None)
tids[tid] = True
if tid is not None:
if c.has_key("code"):
if len(c["code"]) > 4:
code_dict[tid] = fp.decode_code_string(c["code"])
fp.ingest(code_dict, local=True)
lp = {}
for r in _local_bigeval.keys():
if tids.has_key(r):
lp[r] = _local_bigeval[r]
_local_bigeval = lp
local = True
if single is not None:
test_single(single,
local=local,
start=start,
duration=duration,
bitrate=bitrate,
volume=volume,
lowpass_freq=lowpass,
encode_to=encoder,
downsample_to_22=downsample,
channels=channels)
else:
results = test(how_many,
diag=diag,
local=local,
no_shuffle=no_shuffle,
start=start,
duration=duration,
bitrate=bitrate,
volume=volume,
lowpass_freq=lowpass,
encode_to=encoder,
downsample_to_22=downsample,
channels=channels)
prf(results)
dpwe(results)
def main(argv):
global _local_bigeval, _new_music_files
global _new_queries, _old_queries, _total_queries
single = None
how_many = None
start = 0
duration = 30
bitrate = 128
volume = -1
lowpass = -1
decoder = "mpg123"
encoder = "wav"
local = None
diag = False
channels = 2
downsample = False
decoder = "mpg123"
testfile = os.path.join(os.path.dirname(__file__), 'bigeval.json')
newfile = "new_music"
no_shuffle = False
try:
opts, args = getopt.getopt(argv, "1:c:s:d:D:b:v:l:L:e:B:t:n:pm2h",
["single=","count=","start=","duration=", "decoder=","bitrate=","volume=","lowpass="******"encoder=","print","mono","local=", "test=", "new=","22kHz","help","no-shuffle"])
except getopt.GetoptError:
usage()
sys.exit(2)
for opt,arg in opts:
if opt in ("-1","--single"):
single = arg
if opt in ("-c","--count"):
how_many = int(arg)
if opt in ("-s","--start"):
start = int(arg)
if opt in ("-d","--duration"):
duration = int(arg)
if opt in ("-D","--decoder"):
decoder = arg
if opt in ("-b","--bitrate"):
bitrate = int(arg)
if opt in ("-v","--volume"):
volume = int(arg)
if opt in ("-l","--lowpass"):
lowpass = int(arg)
if opt in ("-e","--encoder"):
encoder = arg
if opt in ("-L","--local"):
local = arg
if opt in ("-p","--print"):
diag = True
if opt in ("-m","--mono"):
channels = 1
if opt in ("-2","--22kHz"):
downsample = True
if opt in ("-B","--binary"):
if not os.path.exists(arg):
print "Binary %s not found. Exiting." % arg
sys.exit(2)
config.CODEGEN_BINARY_OVERRIDE = arg
if opt in ("-n","--new"):
newfile = arg
if opt in ("-t","--test"):
testfile = arg
if opt == "--no-shuffle":
no_shuffle = True
if opt in ("-h","--help"):
usage()
sys.exit(2)
if (single is None) and (how_many is None):
print >>sys.stderr, "Run in single mode (-1) or say how many files to test (-c)"
usage()
sys.exit(2)
if testfile.lower() == "none" and newfile.lower() == "none" and single is None:
# If both are none, we can't run
print >>sys.stderr, "Can't run with no datafiles. Skip --test, --new or add -1"
sys.exit(2)
if testfile.lower() == "none":
_local_bigeval = {}
else:
if not os.path.exists(testfile):
print >>sys.stderr, "Cannot find bigeval.json. did you run fastingest with the -b flag?"
sys.exit(1)
_local_bigeval = json.load(open(testfile,'r'))
if newfile.lower() == "none" or not os.path.exists(newfile):
_new_music_files = []
else:
_new_music_files = open(newfile,'r').read().split('\n')
_new_queries = float(len(_new_music_files))
_old_queries = float(len(_local_bigeval.keys()))
_total_queries = _new_queries + _old_queries
if local is None:
local = False
else:
# ingest
codes = json.load(open(local,'r'))
_reversed_bigeval = dict( (_local_bigeval[k], k) for k in _local_bigeval)
code_dict = {}
tids = {}
for c in codes:
fn = c["metadata"]["filename"]
tid = _reversed_bigeval.get(fn, None)
tids[tid] = True
if tid is not None:
if c.has_key("code"):
if len(c["code"]) > 4:
code_dict[tid] = fp.decode_code_string(c["code"])
fp.ingest(code_dict, local=True)
lp = {}
for r in _local_bigeval.keys():
if tids.has_key(r):
lp[r] = _local_bigeval[r]
_local_bigeval = lp
local = True
if single is not None:
test_single(single, local=local, start=start, duration = duration, bitrate = bitrate, volume = volume, lowpass_freq = lowpass, encode_to=encoder, downsample_to_22 = downsample, channels = channels)
else:
results = test(how_many, diag = diag, local=local, no_shuffle=no_shuffle, start=start, duration = duration, bitrate = bitrate, volume = volume, lowpass_freq = lowpass, encode_to=encoder, downsample_to_22 = downsample, channels = channels)
prf(results)
dpwe(results)
def main():
if not len(sys.argv) == 4:
print "usage: python little_eval.py [database_list | disk] query_list [limit]"
sys.exit()
fp_codes = []
limit = int(sys.argv[3])
if sys.argv[1] == "disk":
fp.local_load("disk.pkl")
else:
database_list = open(sys.argv[1]).read().split("\n")[0:limit]
for line in database_list:
(track_id, file) = line.split(" ### ")
print track_id
# TODO - use threaded codegen
j = codegen(file, start=-1, duration=-1)
if len(j):
code_str = fp.decode_code_string(j[0]["code"])
meta = j[0]["metadata"]
l = meta["duration"] * 1000
a = meta["artist"]
r = meta["release"]
t = meta["title"]
v = meta["version"]
fp_codes.append({
"track_id": track_id,
"fp": code_str,
"length": str(l),
"codever": str(round(v, 2)),
"artist": a,
"release": r,
"track": t
})
fp.ingest(fp_codes, local=True)
fp.local_save("disk.pkl")
counter = 0
actual_win = 0
original_win = 0
bm_win = 0
query_list = open(sys.argv[2]).read().split("\n")[0:limit]
for line in query_list:
(track_id, file) = line.split(" ### ")
print track_id
j = codegen(munge(file))
if len(j):
counter += 1
response = fp.query_fp(fp.decode_code_string(j[0]["code"]),
rows=30,
local=True,
get_data=True)
(winner_actual, winner_original) = get_winners(
fp.decode_code_string(j[0]["code"]), response, elbow=8)
winner_actual = winner_actual.split("-")[0]
winner_original = winner_original.split("-")[0]
response = fp.best_match_for_query(j[0]["code"], local=True)
if (response.TRID == track_id):
bm_win += 1
if (winner_actual == track_id):
actual_win += 1
if (winner_original == track_id):
original_win += 1
print "%d / %d actual (%2.2f%%) %d / %d original (%2.2f%%) %d / %d bm (%2.2f%%)" % (actual_win, counter, (float(actual_win)/float(counter))*100.0, \
original_win, counter, (float(original_win)/float(counter))*100.0, \
bm_win, counter, (float(bm_win)/float(counter))*100.0)
def main(argv):
global _local_bigeval
single = None
how_many = None
start = 0
duration = 30
bitrate = 128
volume = -1
lowpass = -1
encoder = "wav"
local = None
diag = False
channels = 2
downsample = False
try:
opts, args = getopt.getopt(argv, "1:c:s:d:b:v:l:L:e:B:pm2h", ["single=","count=","start=","duration=","bitrate=","volume=","lowpass="******"encoder=","print","mono","local=","22kHz","help"])
except getopt.GetoptError:
usage()
sys.exit(2)
for opt,arg in opts:
if opt in ("-1","--single"):
single = arg
if opt in ("-c","--count"):
how_many = int(arg)
if opt in ("-s","--start"):
start = int(arg)
if opt in ("-d","--duration"):
duration = int(arg)
if opt in ("-b","--bitrate"):
bitrate = int(arg)
if opt in ("-v","--volume"):
volume = int(arg)
if opt in ("-l","--lowpass"):
lowpass = int(arg)
if opt in ("-e","--encoder"):
encoder = arg
if opt in ("-L","--local"):
local = arg
if opt in ("-p","--print"):
diag = True
if opt in ("-m","--mono"):
channels = 1
if opt in ("-2","--22kHz"):
downsample = True
if opt in ("-B","--binary"):
if not os.path.exists(arg):
print "Binary %s not found. Exiting." % arg
sys.exit(2)
config.CODEGEN_BINARY_OVERRIDE = arg
if opt in ("-h","--help"):
usage()
if (single is None) and (how_many is None):
usage()
sys.exit(2)
if (single is not None) and (diag is not False):
usage()
sys.exit(2)
if local is None:
local = False
else:
# ingest
codes = json.load(open(local,'r'))
_reversed_bigeval = dict( (_local_bigeval[k], k) for k in _local_bigeval)
code_dict = {}
tids = {}
for c in codes:
fn = c["metadata"]["filename"]
tid = _reversed_bigeval.get(fn, None)
tids[tid] = True
if tid is not None:
if c.has_key("code"):
if len(c["code"]) > 4:
code_dict[tid] = fp.decode_code_string(c["code"])
fp.ingest(code_dict, local=True)
lp = {}
for r in _local_bigeval.keys():
if tids.has_key(r):
lp[r] = _local_bigeval[r]
_local_bigeval = lp
local = True
if single is not None:
test_single(single, local=local, start=start, duration = duration, bitrate = bitrate, volume = volume, lowpass_freq = lowpass, encode_to=encoder, downsample_to_22 = downsample, channels = channels)
else:
results = test(how_many, diag = diag, local=local, start=start, duration = duration, bitrate = bitrate, volume = volume, lowpass_freq = lowpass, encode_to=encoder, downsample_to_22 = downsample, channels = channels)
prf(results)
dpwe(results)