예제 #1
0
파일: audio.py 프로젝트: EQ4/DRR
def stitch(file_list, force_stitch=False):
  # Takes a list of files and then attempt to seamlessly stitch them 
  # together by looking at their signature checksums of the data payload in the blocks.
  duration = 0

  start_index = 0

  while start_index < len(file_list):
    first = file_list[start_index]
    res = cloud.get(first['name'], do_open=False)
    start_index += 1
    if res: break

  # I can't really figure out what this code is about but I do know that it will fail
  # in cases where there's only one file to slice.  So there's a small check to make 
  # it ok for that case
  if start_index == len(file_list) and len(file_list) > 1:
    logging.error("Unable to find any files matching in the list for stitching.")
    return None

  siglist, offset = signature(first['name'])

  # print first, start_index
  first['siglist'] = siglist
  first['offset'] = offset

  end_byte = first['offset'][0]
  if len(first['offset']) > 2:
    end_byte = first['offset'][-2]

  else:
    logging.warn("%s is only %d frames" % (first['name'], len(offset)))

  args = [{
    'name': first['name'], 
    # We don't let the first byte be the beginning because we want
    # to produce valid files.
    'start_byte': first['offset'][0], 
    'start_offset': 0,
    'end_byte': end_byte,
    'start_minute': 0,
    'duration_sec': (len(first['offset']) - 1) * _FRAME_LENGTH
  }]

  duration += len(first['offset']) * _FRAME_LENGTH

  for second in file_list[start_index:]:
    res = cloud.get(second['name'], do_open=False)

    if not res:
      continue

    siglist, offset = signature(second['name'])

    second['siglist'] = siglist
    second['offset'] = offset

    is_found = True

    pos = -1
    try:
      while True:
        # The pos will be the same frame in the second stream as the first.
        pos = second['siglist'].index(first['siglist'][-2], pos + 1)

        is_found = True
        for i in xrange(5, 1, -1):
          if second['siglist'][pos - i + 2] != first['siglist'][-i]:
            is_found = False
            logging.warn("Indices @%d do not match between %s and %s" % (pos, first['name'], second['name']))
            break

        # If we got here it means that everything matches
        if is_found: break 
        else: continue

    except Exception as exc:
      logging.warn("Cannot find indices between %s and %s" % (first['name'], second['name']))
      pos = 1

    if is_found or force_stitch:
      # Since the pos was the same frame, if we use that then we essentially
      # use the same frame twice ... so we need to start one ahead of it.  The
      # easiest way to do this is just to increment the pos var.
      if is_found:
        pos += 1
        """
        import binascii

        print "----"
        for i in xrange(-4, 4):
          if i < 2:
            p1 = binascii.b2a_hex(first['siglist'][i - 2])

            if i == 0:
              p1 += '*'

          else:
            p1 = ''

          print "%s %s" % (binascii.b2a_hex(second['siglist'][pos + i]), p1)
        print args 
        """

      end_byte = second['offset'][0]
      if len(second['offset']) > 2:
        end_byte = second['offset'][-2]

      else: 
        logging.warn("%s is only %d frames" % (second['name'], len(second['offset'])))

      args.append({
        'name': second['name'], 
        'start_byte': second['offset'][pos], 
        'end_byte': end_byte,
        'start_offset': pos,
        'start_minute': (pos * _FRAME_LENGTH) / 60.0,
        'duration_sec': (len(second['offset']) - pos - 1) * _FRAME_LENGTH
      })

      duration += (len(second['offset']) - pos - 1) * _FRAME_LENGTH
      first = second
      continue

    break

  return args
예제 #2
0
파일: audio.py 프로젝트: EQ4/DRR
def list_slice(list_in, name_out, duration_sec, start_sec=0, do_confirm=False):
  # Takes some stitch list, list_in and then create a new one based on the start and end times 
  # by finding the closest frames and just doing an extraction.
  #
  # Setting the duration as None is equivalent to a forever stream 
  pid = misc.change_proc_name("%s-audioslice" % misc.config['callsign'])

  out = open(name_out, 'wb+')
  buf_confirm = None
  
  # print 'slice', duration_sec, start_sec
  for ix in range(0, len(list_in)):
    item = list_in[ix]

    # get the regular map
    siglist, offset = signature(item['name'])

    if ix == len(list_in) - 1:
      frame_end = min(int(ceil(duration_sec / _FRAME_LENGTH)), len(offset) - 1)

    else:
      frame_end = len(offset) - 1

    if ix == 0:
      frame_start = min(max(int(start_sec / _FRAME_LENGTH), 0), len(offset) - 1)
      duration_sec -= (item['duration_sec'] - start_sec)

    else:
      frame_start = item['start_offset']
      duration_sec -= item['duration_sec'] 

    # try and get the mp3
    fin = cloud.get(item['name'])

    if fin:
      fin.seek(offset[frame_start])

      if do_confirm and buf_confirm:
        fin.seek(-16, 1)
        buf = fin.read(16)
        if buf != buf_confirm:
          logging.warn("Slicing error at %d of %s" % (fin.tell(), item['name']))

      # print 'off---',frame_end, frame_start, len(offset)
      buf = fin.read(offset[frame_end] - offset[frame_start])
      out.write(buf)

      if do_confirm:
        buf_confirm = buf[-16]

      fin.close()

    # If we fail to get the mp3 file then we can suppose that
    # the map file is bad so we just wince and remove it.
    else:
      os.unlink(item['name'])
      logging.warn("Unable to find %s's corresponding mp3, deleting" % item['name'])

  out.close()

  # If we failed to do anything this is a tragedy
  # and we just dump the file
  #
  # We take files under some really nominal threshold as being invalid.
  if os.path.getsize(name_out) < 1000:
    logging.warn("Unable to create %s - no valid slices" % name_out)
    os.unlink(name_out)
예제 #3
0
파일: audio.py 프로젝트: EQ4/DRR
def list_slice_stream(start_info, start_sec):
  # This is part of the /live/time feature ... this streams files hopping from one to the next
  # in a live manner ... it constructs things while running ... hopping to the next stream in real time.
  pid = misc.change_proc_name("%s-audiostream" % misc.config['callsign'])
  block_count = 0

  current_info = start_info

  # get the regular map so we know where to start from
  siglist, offset = signature(current_info['name'])
  start_frame = min(max(int(start_sec / _FRAME_LENGTH), 0), len(offset) - 1)
  start_byte = offset[start_frame]

  while True:
    stream_handle = cloud.get(current_info['name'])
    stream_handle.seek(start_byte)
    sig, offset = signature(stream_handle)
    logging.debug("-- opening %s %d %d %d" % (current_info['name'], current_info['size'], stream_handle.tell(), start_byte) )

    # This helps us determine when we are at EOF ... which
    # we basically define as a number of seconds without any
    # valid read.
    times_none = 0
    block_count = 0
    read_size = 0

    while True:
      # So we want to make sure that we only send out valid, 
      # non-corrupt mp3 blocks that start and end
      # at reasonable intervals.
      if len(offset) > 1:
        read_size = offset[1] - offset[0]
        offset.pop(0)

        block = stream_handle.read(read_size)
        block_count += 1
        times_none = 0 
        yield block

      else:  
        times_none += 1
        if times_none > 20:
          break
    
        elif times_none > 1:
          #print stream_handle.tell(), current_info['size'], times_none, len(block)
          # See if there's a next file that we can immediately go to
          next_info, offset = cloud.get_next(current_info)
          if next_info: break

        # We wait 1/2 second and then try this process again, hopefully
        # the disk has sync'd and we have more data
        sleep(0.5)
        sig, offset = signature(stream_handle)

      
    logging.debug("-- closing %s %d %d %d %d" % (current_info['name'], current_info['size'], stream_handle.tell(), block_count, (stream_handle.tell() - start_byte) / (128000 / 8) / 60.0))
    pos = stream_handle.tell() 
    stream_handle.close()

    # If we are here that means that we ran out of data on our current
    # file.  The first things we should do is see if there is a next file
    next_info, offset = cloud.get_next(current_info)

    if next_info:

      # If there is we find the stitching point
      args = stitch([current_info, next_info], force_stitch=True)
      print args, pos

      # We make it our current file
      current_info = next_info

      # Now we can assume that our args[1] is going to have all
      # the information pertaining to where the new file should pick
      # up from - all we really need is the start_byte
      if len(args) == 2:
        start_byte = args[1]['start_byte'] 
        # print "Starting at ", start_byte

      else:
        logging.warn("Live stitching failed")
        break

    else:
      # Otherwise we have to bail
      break
예제 #4
0
파일: audio.py 프로젝트: EQ4/DRR
def stitch(file_list, force_stitch=False):
    # Takes a list of files and then attempt to seamlessly stitch them
    # together by looking at their signature checksums of the data payload in the blocks.
    duration = 0

    start_index = 0

    while start_index < len(file_list):
        first = file_list[start_index]
        res = cloud.get(first['name'], do_open=False)
        start_index += 1
        if res: break

    # I can't really figure out what this code is about but I do know that it will fail
    # in cases where there's only one file to slice.  So there's a small check to make
    # it ok for that case
    if start_index == len(file_list) and len(file_list) > 1:
        logging.error(
            "Unable to find any files matching in the list for stitching.")
        return None

    siglist, offset = signature(first['name'])

    # print first, start_index
    first['siglist'] = siglist
    first['offset'] = offset

    end_byte = first['offset'][0]
    if len(first['offset']) > 2:
        end_byte = first['offset'][-2]

    else:
        logging.warn("%s is only %d frames" % (first['name'], len(offset)))

    args = [{
        'name': first['name'],
        # We don't let the first byte be the beginning because we want
        # to produce valid files.
        'start_byte': first['offset'][0],
        'start_offset': 0,
        'end_byte': end_byte,
        'start_minute': 0,
        'duration_sec': (len(first['offset']) - 1) * _FRAME_LENGTH
    }]

    duration += len(first['offset']) * _FRAME_LENGTH

    for second in file_list[start_index:]:
        res = cloud.get(second['name'], do_open=False)

        if not res:
            continue

        siglist, offset = signature(second['name'])

        second['siglist'] = siglist
        second['offset'] = offset

        is_found = True

        pos = -1
        try:
            while True:
                # The pos will be the same frame in the second stream as the first.
                pos = second['siglist'].index(first['siglist'][-2], pos + 1)

                is_found = True
                for i in xrange(5, 1, -1):
                    if second['siglist'][pos - i + 2] != first['siglist'][-i]:
                        is_found = False
                        logging.warn(
                            "Indices @%d do not match between %s and %s" %
                            (pos, first['name'], second['name']))
                        break

                # If we got here it means that everything matches
                if is_found: break
                else: continue

        except Exception as exc:
            logging.warn("Cannot find indices between %s and %s" %
                         (first['name'], second['name']))
            pos = 1

        if is_found or force_stitch:
            # Since the pos was the same frame, if we use that then we essentially
            # use the same frame twice ... so we need to start one ahead of it.  The
            # easiest way to do this is just to increment the pos var.
            if is_found:
                pos += 1
                """
        import binascii

        print "----"
        for i in xrange(-4, 4):
          if i < 2:
            p1 = binascii.b2a_hex(first['siglist'][i - 2])

            if i == 0:
              p1 += '*'

          else:
            p1 = ''

          print "%s %s" % (binascii.b2a_hex(second['siglist'][pos + i]), p1)
        print args 
        """

            end_byte = second['offset'][0]
            if len(second['offset']) > 2:
                end_byte = second['offset'][-2]

            else:
                logging.warn("%s is only %d frames" %
                             (second['name'], len(second['offset'])))

            args.append({
                'name':
                second['name'],
                'start_byte':
                second['offset'][pos],
                'end_byte':
                end_byte,
                'start_offset':
                pos,
                'start_minute': (pos * _FRAME_LENGTH) / 60.0,
                'duration_sec':
                (len(second['offset']) - pos - 1) * _FRAME_LENGTH
            })

            duration += (len(second['offset']) - pos - 1) * _FRAME_LENGTH
            first = second
            continue

        break

    return args
예제 #5
0
파일: audio.py 프로젝트: EQ4/DRR
def list_slice(list_in, name_out, duration_sec, start_sec=0, do_confirm=False):
    # Takes some stitch list, list_in and then create a new one based on the start and end times
    # by finding the closest frames and just doing an extraction.
    #
    # Setting the duration as None is equivalent to a forever stream
    pid = misc.change_proc_name("%s-audioslice" % misc.config['callsign'])

    out = open(name_out, 'wb+')
    buf_confirm = None

    # print 'slice', duration_sec, start_sec
    for ix in range(0, len(list_in)):
        item = list_in[ix]

        # get the regular map
        siglist, offset = signature(item['name'])

        if ix == len(list_in) - 1:
            frame_end = min(int(ceil(duration_sec / _FRAME_LENGTH)),
                            len(offset) - 1)

        else:
            frame_end = len(offset) - 1

        if ix == 0:
            frame_start = min(max(int(start_sec / _FRAME_LENGTH), 0),
                              len(offset) - 1)
            duration_sec -= (item['duration_sec'] - start_sec)

        else:
            frame_start = item['start_offset']
            duration_sec -= item['duration_sec']

        # try and get the mp3
        fin = cloud.get(item['name'])

        if fin:
            fin.seek(offset[frame_start])

            if do_confirm and buf_confirm:
                fin.seek(-16, 1)
                buf = fin.read(16)
                if buf != buf_confirm:
                    logging.warn("Slicing error at %d of %s" %
                                 (fin.tell(), item['name']))

            # print 'off---',frame_end, frame_start, len(offset)
            buf = fin.read(offset[frame_end] - offset[frame_start])
            out.write(buf)

            if do_confirm:
                buf_confirm = buf[-16]

            fin.close()

        # If we fail to get the mp3 file then we can suppose that
        # the map file is bad so we just wince and remove it.
        else:
            os.unlink(item['name'])
            logging.warn("Unable to find %s's corresponding mp3, deleting" %
                         item['name'])

    out.close()

    # If we failed to do anything this is a tragedy
    # and we just dump the file
    #
    # We take files under some really nominal threshold as being invalid.
    if os.path.getsize(name_out) < 1000:
        logging.warn("Unable to create %s - no valid slices" % name_out)
        os.unlink(name_out)
예제 #6
0
파일: audio.py 프로젝트: EQ4/DRR
def list_slice_stream(start_info, start_sec):
    # This is part of the /live/time feature ... this streams files hopping from one to the next
    # in a live manner ... it constructs things while running ... hopping to the next stream in real time.
    pid = misc.change_proc_name("%s-audiostream" % misc.config['callsign'])
    block_count = 0

    current_info = start_info

    # get the regular map so we know where to start from
    siglist, offset = signature(current_info['name'])
    start_frame = min(max(int(start_sec / _FRAME_LENGTH), 0), len(offset) - 1)
    start_byte = offset[start_frame]

    while True:
        stream_handle = cloud.get(current_info['name'])
        stream_handle.seek(start_byte)
        sig, offset = signature(stream_handle)
        logging.debug("-- opening %s %d %d %d" %
                      (current_info['name'], current_info['size'],
                       stream_handle.tell(), start_byte))

        # This helps us determine when we are at EOF ... which
        # we basically define as a number of seconds without any
        # valid read.
        times_none = 0
        block_count = 0
        read_size = 0

        while True:
            # So we want to make sure that we only send out valid,
            # non-corrupt mp3 blocks that start and end
            # at reasonable intervals.
            if len(offset) > 1:
                read_size = offset[1] - offset[0]
                offset.pop(0)

                block = stream_handle.read(read_size)
                block_count += 1
                times_none = 0
                yield block

            else:
                times_none += 1
                if times_none > 20:
                    break

                elif times_none > 1:
                    #print stream_handle.tell(), current_info['size'], times_none, len(block)
                    # See if there's a next file that we can immediately go to
                    next_info, offset = cloud.get_next(current_info)
                    if next_info: break

                # We wait 1/2 second and then try this process again, hopefully
                # the disk has sync'd and we have more data
                sleep(0.5)
                sig, offset = signature(stream_handle)

        logging.debug("-- closing %s %d %d %d %d" %
                      (current_info['name'], current_info['size'],
                       stream_handle.tell(), block_count,
                       (stream_handle.tell() - start_byte) /
                       (128000 / 8) / 60.0))
        pos = stream_handle.tell()
        stream_handle.close()

        # If we are here that means that we ran out of data on our current
        # file.  The first things we should do is see if there is a next file
        next_info, offset = cloud.get_next(current_info)

        if next_info:

            # If there is we find the stitching point
            args = stitch([current_info, next_info], force_stitch=True)
            print args, pos

            # We make it our current file
            current_info = next_info

            # Now we can assume that our args[1] is going to have all
            # the information pertaining to where the new file should pick
            # up from - all we really need is the start_byte
            if len(args) == 2:
                start_byte = args[1]['start_byte']
                # print "Starting at ", start_byte

            else:
                logging.warn("Live stitching failed")
                break

        else:
            # Otherwise we have to bail
            break