def get_state_and_file(outdir):
state_file = os.path.join(outdir, 'state.log')
state = {}
file = None
if not os.path.exists(state_file):
file = pycommons.open_file(state_file, 'wb')
file.write('{}')
file.close()
else:
with pycommons.open_file(state_file, 'rb') as f:
state = json.loads(f.read())
state_file = open(state_file, 'wb')
return state, state_file
def print_bpm(path, **kwargs):
if not path.endswith('.bpm'):
path += '.bpm.gz'
with pycommons.open_file(path, 'rb', True) as f:
bpm = json.loads(f.read())
print '{} -> {}'.format(path, np.median(bpm['bpm']))
plot(path, bpm['bpm'], **bpm)
def on_exit():
outdir = args.outdir
fpath = os.path.join(outdir, 'state.log')
if len(state) > 0:
with pycommons.open_file(fpath, 'wb') as f:
f.write(json.dumps(state))
print 'Wrote state before exit'
def __process(tempo, energy, path):
with pycommons.open_file(path, 'rb') as f:
data = json.loads(f.read())
t = data['tempo']
e = data['energy']
tempo.append(t)
energy.append(e)
return [t, e]
def dump_callback(path):
pitches, confidences = process(path)
d = {
'file': path,
'pitches': [float(x) for x in pitches],
'confidences': [float(x) for x in confidences],
}
outpath = path + '.pitch.gz'
with pycommons.open_file(outpath, 'wb', True) as f:
f.write(json.dumps(d))
def process(path, queue):
name, _ = os.path.splitext(os.path.basename(path))
# Split name by -
match = pattern.match(name)
try:
assert match, 'Did not match: {}'.format(name)
gd = match.groupdict()
with pycommons.open_file(path, 'rb') as f:
data = json.loads(f.read())
data['artist'] = gd['artist']
data['track'] = gd['track']
with pycommons.open_file(path, 'wb') as f:
f.write(json.dumps(data))
if queue is not None:
queue.put((None))
except Exception, e:
if queue is not None:
tb = traceback.format_exc()
queue.put((path, tb))
raise e
def dump_bpm(path, **kwargs):
outpath = path + '.bpm.gz'
allbpm, total_frames, win_s, samplerate, hop_s = get_file_all_bpm(path)
d = {
'file': path,
'bpm': list(allbpm),
'total_frames': total_frames,
'win_s': win_s,
'samplerate': samplerate,
'hop_s': hop_s,
}
with pycommons.open_file(outpath, 'wb', True) as f:
f.write(json.dumps(d))
def main(argv):
parser = setup_parser()
args = parser.parse_args(argv[1:])
if not os.path.exists(args.jobsfile):
logger.error("Jobsfile '%s' does not exist" % (args.jobsfile))
sys.exit(-1)
else:
Container.NJOBS = 0
with pycommons.open_file(args.jobsfile, 'rb') as f:
for line in f:
line = line.strip()
if line != '':
Container.JOBS.put(line)
Container.NJOBS += 1
server(args.port)
def server(port, secret, out=None):
logger.info("Starting server ...")
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if out:
out = pycommons.open_file(output, 'wb')
generic_logging.add_file_handler(out, logger)
else:
out = sys.stdout
try:
server_socket.bind(('', port))
except socket.error as e:
logger.error('Bind failed! :' + e[1])
sys.exit(-1)
server_socket.listen(10)
while 1:
sock, addr = server_socket.accept()
# print str(addr)
length = struct.unpack('>Q', common.sock_read(sock, 8))[0]
logger.info("Request length: %d" % (length))
msg_buf = common.sock_read(sock, length)
request = protocol_pb2.Request()
request.ParseFromString(msg_buf)
if request.secret != common.sha256(secret):
response = protocol_pb2.Response()
response.type = protocol_pb2.Response.GENERIC
response.status = protocol_pb2.ERROR
response.error = "Invalid secret"
send_response(sock, response)
sock.close()
continue
response = protocol_pb2.Response()
if request.type == protocol_pb2.Request.KEY_REQUEST:
handle_key_request(sock, request.keyRequest, response)
send_response(sock, response)
sock.close()
def server(port, secret, out=None):
logger.info("Starting server ...")
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
if out:
out = pycommons.open_file(output, 'wb')
generic_logging.add_file_handler(out, logger)
else:
out = sys.stdout
try:
server_socket.bind(('', port))
except socket.error as e:
logger.error('Bind failed! :' + e[1])
sys.exit(-1)
server_socket.listen(10)
while 1:
sock, addr = server_socket.accept()
# print str(addr)
length = struct.unpack('>Q', common.sock_read(sock, 8))[0]
logger.info("Request length: %d" % (length))
msg_buf = common.sock_read(sock, length)
request = protocol_pb2.Request()
request.ParseFromString(msg_buf)
if request.secret != common.sha256(secret):
response = protocol_pb2.Response()
response.type = protocol_pb2.Response.GENERIC
response.status = protocol_pb2.ERROR
response.error = "Invalid secret"
send_response(sock, response)
sock.close()
continue
response = protocol_pb2.Response()
if request.type == protocol_pb2.Request.KEY_REQUEST:
handle_key_request(sock, request.keyRequest, response)
send_response(sock, response)
sock.close()
def print_callback(path):
if path.endswith('.mp3'):
path += '.pitch'
with pycommons.open_file(path, 'rb', True) as f:
d = json.loads(f.read())
json.dumps(d, indent=2)
def process(path, outdir, regex, sampling):
if not os.path.exists(outdir):
os.makedirs(outdir)
global state
# Get a file that should exist in the outdir
# this file is used for saving state between multiple
# runs of this script
state, state_file = get_state_and_file(outdir)
s = spotify.Spotify()
s.authenticate()
entry = FileEntry(path, None)
entry.build(regex=regex)
for file in entry.get_files():
digest = hashlib.md5(open(file.path(), 'rb').read()).hexdigest()
if state.get(digest, None) is not None:
continue
try:
data = mutagenwrapper.read_tags(file.path())
artist = data.get('artist')[0]
title = data.get('title')[0]
if artist == '' or title == '':
logger.error('Failed on file {}'.format(file.path()))
continue
print '{} - {}'.format(artist, title)
params = {
'q': 'artist:{} title:{}'.format(artist, title),
'type': 'track',
'limit': 1
}
except Exception, e:
tb = traceback.format_exc()
print tb
try:
search = s.search(params)
item0 = search['tracks']['items'][0]
trackid = item0['id']
artist = item0['artists'][0]['name']
track = item0['name']
features = s.audio_features(trackid)
analysis = s.audio_analysis(trackid)
features['ub_source_file'] = os.path.abspath(file.path())
analysis['ub_source_file'] = os.path.abspath(file.path())
base = '{} - {}'.format(artist, track)
# XXX: Hack..handle AC/DC
base = base.replace('/', '_')
# Now join with outdir
base = os.path.join(outdir, base)
features_file = '{}.features.gz'.format(base)
analysis_file = '{}.analysis.gz'.format(base)
with pycommons.open_file(features_file, 'wb', True) as f:
f.write(json.dumps(features))
with pycommons.open_file(analysis_file, 'wb', True) as f:
f.write(json.dumps(analysis))
state[digest] = True
except Exception, e:
logger.error("Could not process file {}: {}".format(
file.path(), e))