def stitch(file_list, force_stitch=False):
# Takes a list of files and then attempt to seamlessly stitch them
# together by looking at their signature checksums of the data payload in the blocks.
duration = 0
start_index = 0
while start_index < len(file_list):
first = file_list[start_index]
res = cloud.get(first['name'], do_open=False)
start_index += 1
if res: break
# I can't really figure out what this code is about but I do know that it will fail
# in cases where there's only one file to slice. So there's a small check to make
# it ok for that case
if start_index == len(file_list) and len(file_list) > 1:
logging.error("Unable to find any files matching in the list for stitching.")
return None
siglist, offset = signature(first['name'])
# print first, start_index
first['siglist'] = siglist
first['offset'] = offset
end_byte = first['offset'][0]
if len(first['offset']) > 2:
end_byte = first['offset'][-2]
else:
logging.warn("%s is only %d frames" % (first['name'], len(offset)))
args = [{
'name': first['name'],
# We don't let the first byte be the beginning because we want
# to produce valid files.
'start_byte': first['offset'][0],
'start_offset': 0,
'end_byte': end_byte,
'start_minute': 0,
'duration_sec': (len(first['offset']) - 1) * _FRAME_LENGTH
}]
duration += len(first['offset']) * _FRAME_LENGTH
for second in file_list[start_index:]:
res = cloud.get(second['name'], do_open=False)
if not res:
continue
siglist, offset = signature(second['name'])
second['siglist'] = siglist
second['offset'] = offset
is_found = True
pos = -1
try:
while True:
# The pos will be the same frame in the second stream as the first.
pos = second['siglist'].index(first['siglist'][-2], pos + 1)
is_found = True
for i in xrange(5, 1, -1):
if second['siglist'][pos - i + 2] != first['siglist'][-i]:
is_found = False
logging.warn("Indices @%d do not match between %s and %s" % (pos, first['name'], second['name']))
break
# If we got here it means that everything matches
if is_found: break
else: continue
except Exception as exc:
logging.warn("Cannot find indices between %s and %s" % (first['name'], second['name']))
pos = 1
if is_found or force_stitch:
# Since the pos was the same frame, if we use that then we essentially
# use the same frame twice ... so we need to start one ahead of it. The
# easiest way to do this is just to increment the pos var.
if is_found:
pos += 1
"""
import binascii
print "----"
for i in xrange(-4, 4):
if i < 2:
p1 = binascii.b2a_hex(first['siglist'][i - 2])
if i == 0:
p1 += '*'
else:
p1 = ''
print "%s %s" % (binascii.b2a_hex(second['siglist'][pos + i]), p1)
print args
"""
end_byte = second['offset'][0]
if len(second['offset']) > 2:
end_byte = second['offset'][-2]
else:
logging.warn("%s is only %d frames" % (second['name'], len(second['offset'])))
args.append({
'name': second['name'],
'start_byte': second['offset'][pos],
'end_byte': end_byte,
'start_offset': pos,
'start_minute': (pos * _FRAME_LENGTH) / 60.0,
'duration_sec': (len(second['offset']) - pos - 1) * _FRAME_LENGTH
})
duration += (len(second['offset']) - pos - 1) * _FRAME_LENGTH
first = second
continue
break
return args
def list_slice(list_in, name_out, duration_sec, start_sec=0, do_confirm=False):
# Takes some stitch list, list_in and then create a new one based on the start and end times
# by finding the closest frames and just doing an extraction.
#
# Setting the duration as None is equivalent to a forever stream
pid = misc.change_proc_name("%s-audioslice" % misc.config['callsign'])
out = open(name_out, 'wb+')
buf_confirm = None
# print 'slice', duration_sec, start_sec
for ix in range(0, len(list_in)):
item = list_in[ix]
# get the regular map
siglist, offset = signature(item['name'])
if ix == len(list_in) - 1:
frame_end = min(int(ceil(duration_sec / _FRAME_LENGTH)), len(offset) - 1)
else:
frame_end = len(offset) - 1
if ix == 0:
frame_start = min(max(int(start_sec / _FRAME_LENGTH), 0), len(offset) - 1)
duration_sec -= (item['duration_sec'] - start_sec)
else:
frame_start = item['start_offset']
duration_sec -= item['duration_sec']
# try and get the mp3
fin = cloud.get(item['name'])
if fin:
fin.seek(offset[frame_start])
if do_confirm and buf_confirm:
fin.seek(-16, 1)
buf = fin.read(16)
if buf != buf_confirm:
logging.warn("Slicing error at %d of %s" % (fin.tell(), item['name']))
# print 'off---',frame_end, frame_start, len(offset)
buf = fin.read(offset[frame_end] - offset[frame_start])
out.write(buf)
if do_confirm:
buf_confirm = buf[-16]
fin.close()
# If we fail to get the mp3 file then we can suppose that
# the map file is bad so we just wince and remove it.
else:
os.unlink(item['name'])
logging.warn("Unable to find %s's corresponding mp3, deleting" % item['name'])
out.close()
# If we failed to do anything this is a tragedy
# and we just dump the file
#
# We take files under some really nominal threshold as being invalid.
if os.path.getsize(name_out) < 1000:
logging.warn("Unable to create %s - no valid slices" % name_out)
os.unlink(name_out)
def list_slice_stream(start_info, start_sec):
# This is part of the /live/time feature ... this streams files hopping from one to the next
# in a live manner ... it constructs things while running ... hopping to the next stream in real time.
pid = misc.change_proc_name("%s-audiostream" % misc.config['callsign'])
block_count = 0
current_info = start_info
# get the regular map so we know where to start from
siglist, offset = signature(current_info['name'])
start_frame = min(max(int(start_sec / _FRAME_LENGTH), 0), len(offset) - 1)
start_byte = offset[start_frame]
while True:
stream_handle = cloud.get(current_info['name'])
stream_handle.seek(start_byte)
sig, offset = signature(stream_handle)
logging.debug("-- opening %s %d %d %d" % (current_info['name'], current_info['size'], stream_handle.tell(), start_byte) )
# This helps us determine when we are at EOF ... which
# we basically define as a number of seconds without any
# valid read.
times_none = 0
block_count = 0
read_size = 0
while True:
# So we want to make sure that we only send out valid,
# non-corrupt mp3 blocks that start and end
# at reasonable intervals.
if len(offset) > 1:
read_size = offset[1] - offset[0]
offset.pop(0)
block = stream_handle.read(read_size)
block_count += 1
times_none = 0
yield block
else:
times_none += 1
if times_none > 20:
break
elif times_none > 1:
#print stream_handle.tell(), current_info['size'], times_none, len(block)
# See if there's a next file that we can immediately go to
next_info, offset = cloud.get_next(current_info)
if next_info: break
# We wait 1/2 second and then try this process again, hopefully
# the disk has sync'd and we have more data
sleep(0.5)
sig, offset = signature(stream_handle)
logging.debug("-- closing %s %d %d %d %d" % (current_info['name'], current_info['size'], stream_handle.tell(), block_count, (stream_handle.tell() - start_byte) / (128000 / 8) / 60.0))
pos = stream_handle.tell()
stream_handle.close()
# If we are here that means that we ran out of data on our current
# file. The first things we should do is see if there is a next file
next_info, offset = cloud.get_next(current_info)
if next_info:
# If there is we find the stitching point
args = stitch([current_info, next_info], force_stitch=True)
print args, pos
# We make it our current file
current_info = next_info
# Now we can assume that our args[1] is going to have all
# the information pertaining to where the new file should pick
# up from - all we really need is the start_byte
if len(args) == 2:
start_byte = args[1]['start_byte']
# print "Starting at ", start_byte
else:
logging.warn("Live stitching failed")
break
else:
# Otherwise we have to bail
break
def stitch(file_list, force_stitch=False):
# Takes a list of files and then attempt to seamlessly stitch them
# together by looking at their signature checksums of the data payload in the blocks.
duration = 0
start_index = 0
while start_index < len(file_list):
first = file_list[start_index]
res = cloud.get(first['name'], do_open=False)
start_index += 1
if res: break
# I can't really figure out what this code is about but I do know that it will fail
# in cases where there's only one file to slice. So there's a small check to make
# it ok for that case
if start_index == len(file_list) and len(file_list) > 1:
logging.error(
"Unable to find any files matching in the list for stitching.")
return None
siglist, offset = signature(first['name'])
# print first, start_index
first['siglist'] = siglist
first['offset'] = offset
end_byte = first['offset'][0]
if len(first['offset']) > 2:
end_byte = first['offset'][-2]
else:
logging.warn("%s is only %d frames" % (first['name'], len(offset)))
args = [{
'name': first['name'],
# We don't let the first byte be the beginning because we want
# to produce valid files.
'start_byte': first['offset'][0],
'start_offset': 0,
'end_byte': end_byte,
'start_minute': 0,
'duration_sec': (len(first['offset']) - 1) * _FRAME_LENGTH
}]
duration += len(first['offset']) * _FRAME_LENGTH
for second in file_list[start_index:]:
res = cloud.get(second['name'], do_open=False)
if not res:
continue
siglist, offset = signature(second['name'])
second['siglist'] = siglist
second['offset'] = offset
is_found = True
pos = -1
try:
while True:
# The pos will be the same frame in the second stream as the first.
pos = second['siglist'].index(first['siglist'][-2], pos + 1)
is_found = True
for i in xrange(5, 1, -1):
if second['siglist'][pos - i + 2] != first['siglist'][-i]:
is_found = False
logging.warn(
"Indices @%d do not match between %s and %s" %
(pos, first['name'], second['name']))
break
# If we got here it means that everything matches
if is_found: break
else: continue
except Exception as exc:
logging.warn("Cannot find indices between %s and %s" %
(first['name'], second['name']))
pos = 1
if is_found or force_stitch:
# Since the pos was the same frame, if we use that then we essentially
# use the same frame twice ... so we need to start one ahead of it. The
# easiest way to do this is just to increment the pos var.
if is_found:
pos += 1
"""
import binascii
print "----"
for i in xrange(-4, 4):
if i < 2:
p1 = binascii.b2a_hex(first['siglist'][i - 2])
if i == 0:
p1 += '*'
else:
p1 = ''
print "%s %s" % (binascii.b2a_hex(second['siglist'][pos + i]), p1)
print args
"""
end_byte = second['offset'][0]
if len(second['offset']) > 2:
end_byte = second['offset'][-2]
else:
logging.warn("%s is only %d frames" %
(second['name'], len(second['offset'])))
args.append({
'name':
second['name'],
'start_byte':
second['offset'][pos],
'end_byte':
end_byte,
'start_offset':
pos,
'start_minute': (pos * _FRAME_LENGTH) / 60.0,
'duration_sec':
(len(second['offset']) - pos - 1) * _FRAME_LENGTH
})
duration += (len(second['offset']) - pos - 1) * _FRAME_LENGTH
first = second
continue
break
return args
def list_slice(list_in, name_out, duration_sec, start_sec=0, do_confirm=False):
# Takes some stitch list, list_in and then create a new one based on the start and end times
# by finding the closest frames and just doing an extraction.
#
# Setting the duration as None is equivalent to a forever stream
pid = misc.change_proc_name("%s-audioslice" % misc.config['callsign'])
out = open(name_out, 'wb+')
buf_confirm = None
# print 'slice', duration_sec, start_sec
for ix in range(0, len(list_in)):
item = list_in[ix]
# get the regular map
siglist, offset = signature(item['name'])
if ix == len(list_in) - 1:
frame_end = min(int(ceil(duration_sec / _FRAME_LENGTH)),
len(offset) - 1)
else:
frame_end = len(offset) - 1
if ix == 0:
frame_start = min(max(int(start_sec / _FRAME_LENGTH), 0),
len(offset) - 1)
duration_sec -= (item['duration_sec'] - start_sec)
else:
frame_start = item['start_offset']
duration_sec -= item['duration_sec']
# try and get the mp3
fin = cloud.get(item['name'])
if fin:
fin.seek(offset[frame_start])
if do_confirm and buf_confirm:
fin.seek(-16, 1)
buf = fin.read(16)
if buf != buf_confirm:
logging.warn("Slicing error at %d of %s" %
(fin.tell(), item['name']))
# print 'off---',frame_end, frame_start, len(offset)
buf = fin.read(offset[frame_end] - offset[frame_start])
out.write(buf)
if do_confirm:
buf_confirm = buf[-16]
fin.close()
# If we fail to get the mp3 file then we can suppose that
# the map file is bad so we just wince and remove it.
else:
os.unlink(item['name'])
logging.warn("Unable to find %s's corresponding mp3, deleting" %
item['name'])
out.close()
# If we failed to do anything this is a tragedy
# and we just dump the file
#
# We take files under some really nominal threshold as being invalid.
if os.path.getsize(name_out) < 1000:
logging.warn("Unable to create %s - no valid slices" % name_out)
os.unlink(name_out)
def list_slice_stream(start_info, start_sec):
# This is part of the /live/time feature ... this streams files hopping from one to the next
# in a live manner ... it constructs things while running ... hopping to the next stream in real time.
pid = misc.change_proc_name("%s-audiostream" % misc.config['callsign'])
block_count = 0
current_info = start_info
# get the regular map so we know where to start from
siglist, offset = signature(current_info['name'])
start_frame = min(max(int(start_sec / _FRAME_LENGTH), 0), len(offset) - 1)
start_byte = offset[start_frame]
while True:
stream_handle = cloud.get(current_info['name'])
stream_handle.seek(start_byte)
sig, offset = signature(stream_handle)
logging.debug("-- opening %s %d %d %d" %
(current_info['name'], current_info['size'],
stream_handle.tell(), start_byte))
# This helps us determine when we are at EOF ... which
# we basically define as a number of seconds without any
# valid read.
times_none = 0
block_count = 0
read_size = 0
while True:
# So we want to make sure that we only send out valid,
# non-corrupt mp3 blocks that start and end
# at reasonable intervals.
if len(offset) > 1:
read_size = offset[1] - offset[0]
offset.pop(0)
block = stream_handle.read(read_size)
block_count += 1
times_none = 0
yield block
else:
times_none += 1
if times_none > 20:
break
elif times_none > 1:
#print stream_handle.tell(), current_info['size'], times_none, len(block)
# See if there's a next file that we can immediately go to
next_info, offset = cloud.get_next(current_info)
if next_info: break
# We wait 1/2 second and then try this process again, hopefully
# the disk has sync'd and we have more data
sleep(0.5)
sig, offset = signature(stream_handle)
logging.debug("-- closing %s %d %d %d %d" %
(current_info['name'], current_info['size'],
stream_handle.tell(), block_count,
(stream_handle.tell() - start_byte) /
(128000 / 8) / 60.0))
pos = stream_handle.tell()
stream_handle.close()
# If we are here that means that we ran out of data on our current
# file. The first things we should do is see if there is a next file
next_info, offset = cloud.get_next(current_info)
if next_info:
# If there is we find the stitching point
args = stitch([current_info, next_info], force_stitch=True)
print args, pos
# We make it our current file
current_info = next_info
# Now we can assume that our args[1] is going to have all
# the information pertaining to where the new file should pick
# up from - all we really need is the start_byte
if len(args) == 2:
start_byte = args[1]['start_byte']
# print "Starting at ", start_byte
else:
logging.warn("Live stitching failed")
break
else:
# Otherwise we have to bail
break