class AubioEncoder(Processor): implements(IEncoder) abstract() type = 'encoder' def __init__(self, output, streaming=False, overwrite=False): super (AubioEncoder, self).__init__() if isinstance(output, str): import os.path if os.path.isdir(output): raise IOError("Encoder output must be a file, not a directory") elif os.path.isfile(output) and not overwrite: raise IOError( "Encoder output %s exists, but overwrite set to False") self.filename = output else: self.filename = None self.streaming = streaming if not self.filename and not self.streaming: raise Exception('Must give an output') self.eod = False self.metadata = None self.num_samples = 0 @interfacedoc def setup(self, channels=None, samplerate=None, blocksize=None, totalframes=None): super(AubioEncoder, self).setup( channels, samplerate, blocksize, totalframes) print ('setting up aubio with with', samplerate, blocksize) self.sink = aubio.sink(self.filename, samplerate, channels) @interfacedoc def release(self): if hasattr (self, 'sink'): self.sink.close() delattr(self, 'sink') @interfacedoc def set_metadata(self, metadata): self.metadata = metadata @interfacedoc def process(self, frames, eod=False): self.eod = eod max_write = 4096 indices = range(max_write, frames.shape[0], max_write) for f_slice in np.array_split(frames, indices): write_frames = f_slice.shape[0] self.sink.do_multi(f_slice.T.copy(), write_frames) self.num_samples += write_frames return frames, eod
class Decoder(Processor): """General abstract base class for Decoder """ implements(IDecoder) abstract() type = 'decoder' mimetype = '' output_samplerate = None output_channels = None def __init__(self, start=0, duration=None): super(Decoder, self).__init__() self.uri_start = float(start) if duration: self.uri_duration = float(duration) else: self.uri_duration = duration if start == 0 and duration is None: self.is_segment = False else: self.is_segment = True @interfacedoc def channels(self): return self.output_channels @interfacedoc def samplerate(self): return self.output_samplerate @interfacedoc def blocksize(self): return self.output_blocksize @interfacedoc def totalframes(self): return self.input_totalframes @interfacedoc def release(self): pass @interfacedoc def mediainfo(self): return dict(uri=self.uri, duration=self.uri_duration, start=self.uri_start, is_segment=self.is_segment, samplerate=self.input_samplerate, sha1=self.sha1) @property def sha1(self): return self._sha1 def __del__(self): self.release() @interfacedoc def encoding(self): return self.format().split('/')[-1] @interfacedoc def resolution(self): return self.input_width
class Analyzer(Processor): ''' Generic class for the analyzers ''' type = 'analyzer' implements(IAnalyzer) abstract() def __init__(self): super(Analyzer, self).__init__() def setup(self, channels=None, samplerate=None, blocksize=None, totalframes=None): super(Analyzer, self).setup(channels, samplerate, blocksize, totalframes) # Set default values for result_* attributes # may be overwritten by the analyzer self.result_channels = self.input_channels self.result_samplerate = self.input_samplerate self.result_blocksize = self.input_blocksize self.result_stepsize = self.input_stepsize def add_result(self, result): if not self.uuid() in self.process_pipe.results: self.process_pipe.results[self.uuid()] = AnalyzerResultContainer() self.process_pipe.results[self.uuid()].add(result) @property def results(self): return self.process_pipe.results[self.uuid()] @staticmethod def id(): return "analyzer" @staticmethod def name(): return "Generic analyzer" @staticmethod def unit(): return "" def new_result(self, data_mode='value', time_mode='framewise'): ''' Create a new result Attributes ---------- data_object : MetadataObject id_metadata : MetadataObject audio_metadata : MetadataObject frame_metadata : MetadataObject label_metadata : MetadataObject parameters : dict ''' from datetime import datetime result = AnalyzerResult(data_mode=data_mode, time_mode=time_mode) # Automatically write known metadata result.id_metadata.date = datetime.now().replace( microsecond=0).isoformat(' ') result.id_metadata.version = timeside.core.__version__ result.id_metadata.author = 'TimeSide' result.id_metadata.id = self.id() result.id_metadata.name = self.name() result.id_metadata.description = self.description() result.id_metadata.unit = self.unit() result.id_metadata.proc_uuid = self.uuid() result.audio_metadata.uri = self.mediainfo()['uri'] result.audio_metadata.sha1 = self.mediainfo()['sha1'] result.audio_metadata.start = self.mediainfo()['start'] result.audio_metadata.duration = self.mediainfo()['duration'] result.audio_metadata.is_segment = self.mediainfo()['is_segment'] result.audio_metadata.channels = self.channels() result.parameters = Parameters(json.loads(self.get_parameters())) if time_mode == 'framewise': result.data_object.frame_metadata.samplerate = self.result_samplerate result.data_object.frame_metadata.blocksize = self.result_blocksize result.data_object.frame_metadata.stepsize = self.result_stepsize return result
class GstEncoder(Processor): implements(IEncoder) abstract() type = 'encoder' def __init__(self, output, streaming=False, overwrite=False): super(GstEncoder, self).__init__() if isinstance(output, basestring): import os.path if os.path.isdir(output): raise IOError("Encoder output must be a file, not a directory") elif os.path.isfile(output) and not overwrite: raise IOError( "Encoder output %s exists, but overwrite set to False") self.filename = output else: self.filename = None self.streaming = streaming if not self.filename and not self.streaming: raise Exception('Must give an output') self.end_cond = threading.Condition(threading.Lock()) self.eod = False self.metadata = None self.num_samples = 0 self._chunk_len = 0 @interfacedoc def release(self): if hasattr(self, 'eod') and hasattr(self, 'mainloopthread'): self.end_cond.acquire() while not hasattr(self, 'end_reached'): self.end_cond.wait() self.end_cond.release() if hasattr(self, 'error_msg'): raise IOError(self.error_msg) def __del__(self): self.release() def start_pipeline(self, channels, samplerate): self.pipeline = gst.parse_launch(self.pipe) # store a pointer to appsrc in our encoder object self.src = self.pipeline.get_by_name('src') if self.streaming: import Queue self._streaming_queue = Queue.Queue(QUEUE_SIZE) # store a pointer to appsink in our encoder object self.app = self.pipeline.get_by_name('app') self.app.set_property('max-buffers', GST_APPSINK_MAX_BUFFERS) self.app.set_property("drop", False) self.app.set_property('emit-signals', True) self.app.connect("new-buffer", self._on_new_buffer_streaming) #self.app.connect('new-preroll', self._on_new_preroll_streaming) srccaps = gst.Caps("""audio/x-raw-float, endianness=(int)1234, channels=(int)%s, width=(int)32, rate=(int)%d""" % (int(channels), int(samplerate))) self.src.set_property("caps", srccaps) self.src.set_property('emit-signals', True) self.src.set_property('num-buffers', -1) self.src.set_property('block', False) self.src.set_property('do-timestamp', True) self.bus = self.pipeline.get_bus() self.bus.add_signal_watch() self.bus.connect("message", self._on_message_cb) self.mainloop = gobject.MainLoop() self.mainloopthread = MainloopThread(self.mainloop) self.mainloopthread.start() # start pipeline self.pipeline.set_state(gst.STATE_PLAYING) def _on_message_cb(self, bus, message): t = message.type if t == gst.MESSAGE_EOS: self.end_cond.acquire() if self.streaming: self._streaming_queue.put(gst.MESSAGE_EOS) self.pipeline.set_state(gst.STATE_NULL) self.mainloop.quit() self.end_reached = True self.end_cond.notify() self.end_cond.release() elif t == gst.MESSAGE_ERROR: self.end_cond.acquire() self.pipeline.set_state(gst.STATE_NULL) self.mainloop.quit() self.end_reached = True err, debug = message.parse_error() self.error_msg = "Error: %s" % err, debug self.end_cond.notify() self.end_cond.release() def _on_new_buffer_streaming(self, appsink): # print 'pull-buffer' chunk = appsink.emit('pull-buffer') self._streaming_queue.put(chunk) def _on_new_preroll_streaming(self, appsink): # print 'preroll' chunk = appsink.emit('pull-preroll') self._streaming_queue.put(chunk) @interfacedoc def set_metadata(self, metadata): self.metadata = metadata @interfacedoc def process(self, frames, eod=False): self.eod = eod if eod: self.num_samples += frames.shape[0] else: self.num_samples += self.blocksize() buf = numpy_array_to_gst_buffer(frames, frames.shape[0], self.num_samples, self.samplerate()) self.src.emit('push-buffer', buf) if self.eod: self.src.emit('end-of-stream') return frames, eod def get_stream_chunk(self): if self.streaming: chunk = self._streaming_queue.get(block=True) if chunk == gst.MESSAGE_EOS: return None else: self._streaming_queue.task_done() return chunk else: raise TypeError('function only available in streaming mode')
class Grapher(Processor): ''' Generic abstract class for the graphers ''' type = 'grapher' fft_size = 0x1000 frame_cursor = 0 pixel_cursor = 0 lower_freq = 20 implements(IGrapher) abstract() def __init__(self, width=1024, height=256, bg_color=None, color_scheme='default'): super(Grapher, self).__init__() self.bg_color = bg_color self.color_scheme = color_scheme self.graph = None self.image_width = width self.image_height = height self.bg_color = bg_color self.color_scheme = color_scheme self.previous_x, self.previous_y = None, None @staticmethod def id(): return "generic_grapher" @staticmethod def name(): return "Generic grapher" def set_colors(self, bg_color, color_scheme): self.bg_color = bg_color self.color_color_scheme = color_scheme def setup(self, channels=None, samplerate=None, blocksize=None, totalframes=None): super(Grapher, self).setup(channels, samplerate, blocksize, totalframes) self.sample_rate = samplerate self.higher_freq = self.sample_rate / 2 self.block_size = blocksize self.total_frames = totalframes self.image = Image.new("RGBA", (self.image_width, self.image_height), self.bg_color) self.samples_per_pixel = self.total_frames / float(self.image_width) self.buffer_size = int(round(self.samples_per_pixel, 0)) self.pixels_adapter = FixedSizeInputAdapter(self.buffer_size, 1, pad=False) self.pixels_adapter_totalframes = self.pixels_adapter.blocksize( self.total_frames) self.spectrum = Spectrum(self.fft_size, self.sample_rate, self.block_size, self.total_frames, self.lower_freq, self.higher_freq, numpy.hanning) self.pixel = self.image.load() self.draw = ImageDraw.Draw(self.image) @interfacedoc def render(self, output=None): if output: try: self.image.save(output) except AttributeError: print "Pixel %s x %d" % (self.image_width, self.image_height) self.image.savefig(output, dpi=341) return return self.image def watermark(self, text, font=None, color=(255, 255, 255), opacity=.6, margin=(5, 5)): self.image = im_watermark(self.image, text, color=color, opacity=opacity, margin=margin) def draw_peaks(self, x, peaks, line_color): """Draw 2 peaks at x""" y1 = self.image_height * 0.5 - peaks[0] * (self.image_height - 4) * 0.5 y2 = self.image_height * 0.5 - peaks[1] * (self.image_height - 4) * 0.5 if self.previous_y: self.draw.line([self.previous_x, self.previous_y, x, y1, x, y2], line_color) else: self.draw.line([x, y1, x, y2], line_color) self.draw_anti_aliased_pixels(x, y1, y2, line_color) self.previous_x, self.previous_y = x, y2 def draw_peaks_inverted(self, x, peaks, line_color): """Draw 2 inverted peaks at x""" y1 = self.image_height * 0.5 - peaks[0] * (self.image_height - 4) * 0.5 y2 = self.image_height * 0.5 - peaks[1] * (self.image_height - 4) * 0.5 if self.previous_y and x < self.image_width - 1: if y1 < y2: self.draw.line((x, 0, x, y1), line_color) self.draw.line((x, self.image_height, x, y2), line_color) else: self.draw.line((x, 0, x, y2), line_color) self.draw.line((x, self.image_height, x, y1), line_color) else: self.draw.line((x, 0, x, self.image_height), line_color) self.draw_anti_aliased_pixels(x, y1, y2, line_color) self.previous_x, self.previous_y = x, y1 def draw_anti_aliased_pixels(self, x, y1, y2, color): """ vertical anti-aliasing at y1 and y2 """ y_max = max(y1, y2) y_max_int = int(y_max) alpha = y_max - y_max_int if alpha > 0.0 and alpha < 1.0 and y_max_int + 1 < self.image_height: current_pix = self.pixel[int(x), y_max_int + 1] r = int((1 - alpha) * current_pix[0] + alpha * color[0]) g = int((1 - alpha) * current_pix[1] + alpha * color[1]) b = int((1 - alpha) * current_pix[2] + alpha * color[2]) self.pixel[x, y_max_int + 1] = (r, g, b) y_min = min(y1, y2) y_min_int = int(y_min) alpha = 1.0 - (y_min - y_min_int) if alpha > 0.0 and alpha < 1.0 and y_min_int - 1 >= 0: current_pix = self.pixel[x, y_min_int - 1] r = int((1 - alpha) * current_pix[0] + alpha * color[0]) g = int((1 - alpha) * current_pix[1] + alpha * color[1]) b = int((1 - alpha) * current_pix[2] + alpha * color[2]) self.pixel[x, y_min_int - 1] = (r, g, b) def draw_peaks_contour(self): contour = self.contour.copy() contour = smooth(contour, window_len=16) contour = normalize(contour) # Scaling #ratio = numpy.mean(contour)/numpy.sqrt(2) ratio = 1 contour = normalize(numpy.expm1(contour / ratio)) * (1 - 10**-6) # Spline #contour = cspline1d(contour) #contour = cspline1d_eval(contour, self.x, dx=self.dx1, x0=self.x[0]) if self.symetry: height = int(self.image_height / 2) else: height = self.image_height # Multicurve rotating for i in range(0, self.ndiv): self.previous_x, self.previous_y = None, None bright_color = int(255 * (1 - float(i) / (self.ndiv * 2))) bright_color = 255 - bright_color + self.color_offset #line_color = self.color_lookup[int(self.centroids[j]*255.0)] line_color = (bright_color, bright_color, bright_color) # Linear #contour = contour*(1.0-float(i)/self.ndiv) #contour = contour*(1-float(i)/self.ndiv) # Cosinus contour = contour * \ numpy.arccos(float(i) / self.ndiv) * 2 / numpy.pi #contour = self.contour*(1-float(i)*numpy.arccos(float(i)/self.ndiv)*2/numpy.pi/self.ndiv) #contour = contour + ((1-contour)*2/numpy.pi*numpy.arcsin(float(i)/self.ndiv)) curve = (height - 1) * contour #curve = contour*(height-2)/2+height/2 for x in self.x: x = int(x) y = curve[x] if not x == 0: if not self.symetry: self.draw.line( [self.previous_x, self.previous_y, x, y], line_color) self.draw_anti_aliased_pixels(x, y, y, line_color) else: self.draw.line([ self.previous_x, self.previous_y + height, x, y + height ], line_color) self.draw_anti_aliased_pixels(x, y + height, y + height, line_color) self.draw.line([ self.previous_x, -self.previous_y + height, x, -y + height ], line_color) self.draw_anti_aliased_pixels(x, -y + height, -y + height, line_color) else: if not self.symetry: self.draw.point((x, y), line_color) else: self.draw.point((x, y + height), line_color) self.previous_x, self.previous_y = x, y
class DisplayAnalyzer(Grapher): """ image from analyzer result This is an Abstract base class """ dpi = 72 # Web default value for Telemeta implements(IGrapher) abstract() @interfacedoc def __init__(self, width=1024, height=256, bg_color=(0, 0, 0), color_scheme='default'): super(DisplayAnalyzer, self).__init__(width, height, bg_color, color_scheme) self._result_id = None self._id = NotImplemented self._name = NotImplemented self.image = None self._background = None self._bg_id = '' @interfacedoc def process(self, frames, eod=False): return frames, eod @interfacedoc def post_process(self): pipe_result = self.process_pipe.results analyzer_uuid = self.parents['analyzer'].uuid() analyzer_result = pipe_result[analyzer_uuid][self._result_id] fg_image = analyzer_result._render_PIL( (self.image_width, self.image_height), self.dpi) if self._background: bg_uuid = self.parents['bg_analyzer'].uuid() bg_result = pipe_result[bg_uuid][self._bg_id] bg_image = bg_result._render_PIL( (self.image_width, self.image_height), self.dpi) # convert image to grayscale bg_image = bg_image.convert('LA').convert('RGBA') # Merge background and foreground images from PIL.Image import blend fg_image = blend(fg_image, bg_image, 0.15) self.image = fg_image @classmethod def create(cls, analyzer, analyzer_parameters=None, result_id=None, grapher_id=None, grapher_name=None, background=None, staging=False): if analyzer_parameters is None: analyzer_parameters = {} class NewGrapher(cls): _id = grapher_id _staging = staging _from_analyzer = True _analyzer = analyzer _analyzer_parameters = analyzer_parameters _result_id = result_id _grapher_name = grapher_name implements(IGrapher) @interfacedoc def __init__(self, width=1024, height=256, bg_color=(0, 0, 0), color_scheme='default'): super(NewGrapher, self).__init__(width, height, bg_color, color_scheme) # Add a parent waveform analyzer if background == 'waveform': self._background = True bg_analyzer = timeside.plugins.analyzer.waveform.Waveform() self._bg_id = bg_analyzer.id() self.parents['bg_analyzer'] = bg_analyzer elif background == 'spectrogram': self._background = True bg_analyzer = timeside.plugins.analyzer.spectrogram.Spectrogram( ) self._bg_id = bg_analyzer.id() self.parents['bg_analyzer'] = bg_analyzer else: self._background = None parent_analyzer = analyzer(**analyzer_parameters) self.parents['analyzer'] = parent_analyzer self._result_id = result_id @staticmethod @interfacedoc def id(): return grapher_id @staticmethod @interfacedoc def name(): return grapher_name __doc__ = """Image representing """ + grapher_name NewGrapher.__name__ = grapher_name return NewGrapher
class VampAnalyzer(Analyzer): """Parent Abstract Class for Vamp Analyzer through Vampyhost """ implements(IAnalyzer) abstract() class _Param(HasTraits): pass _schema = { '$schema': 'http://json-schema.org/schema#', 'properties': {}, 'type': 'object' } @store_parameters def __init__(self): super(VampAnalyzer, self).__init__() # Define Vamp plugin key and output # run vamp.list_plugins() to get the list of # available plugins # and vamp.get_outputs_of('plugin_key') # to get the outputs of the corresponding plugin self.plugin_key = None self.plugin_output = None self.output_desc = None # Attributes initialize later during setup self.plugin = None self.out_index = None # TODO: manage several outputs self.vamp_results = [] # Process Attribute self.frame_index = 0 @interfacedoc def setup(self, channels=None, samplerate=None, blocksize=None, totalframes=None): super(VampAnalyzer, self).setup(channels, samplerate, blocksize, totalframes) self.plugin = vampyhost.load_plugin( self.plugin_key, float(self.input_samplerate), vampyhost.ADAPT_INPUT_DOMAIN + vampyhost.ADAPT_BUFFER_SIZE + vampyhost.ADAPT_CHANNEL_COUNT) if not self.plugin.initialise(self.input_channels, self.input_stepsize, self.input_blocksize): raise RuntimeError("Vampy-Host failed to initialise plugin %d" % self.plugin_key) self.out_index = self.plugin.get_output( self.plugin_output)["output_index"] if not self.plugin_output: self.output_desc = self.plugin.get_output(0) self.plugin_output = self.output_desc["identifier"] else: self.output_desc = self.plugin.get_output(self.plugin_output) @staticmethod @interfacedoc def id(): return "vamp_analyzer" @staticmethod @interfacedoc def name(): return "Parent Abstract Class for Vamp Analyzer through Vampyhost" @staticmethod @interfacedoc def unit(): return "" @frames_adapter def process(self, frames, eod=False): timestamp = vampyhost.frame_to_realtime(self.frame_index, self.input_samplerate) results = self.plugin.process_block(frames.T, timestamp) if self.out_index in results: for res in results[self.out_index]: self.vamp_results.append({self.plugin_output: res}) self.frame_index += self.input_stepsize return frames, eod def post_process(self): results = self.plugin.get_remaining_features() if self.out_index in results: for res in results[self.out_index]: self.vamp_results.append({self.plugin_output: res}) shape = deduce_shape(self.output_desc) res_vamp = reshape(self.vamp_results, self.input_samplerate, self.input_stepsize, self.output_desc, shape) self.plugin.unload() self.vamp_results = {shape: res_vamp}
class DisplayAnalyzer(Grapher): """ Builds a PIL image from analyzer result This is an Abstract base class """ dpi = 72 # Web default value for Telemeta implements(IGrapher) abstract() @interfacedoc def __init__(self, width=1024, height=256, bg_color=(0, 0, 0), color_scheme='default'): super(DisplayAnalyzer, self).__init__(width, height, bg_color, color_scheme) self._result_id = None self._id = NotImplemented self._name = NotImplemented @interfacedoc def process(self, frames, eod=False): return frames, eod @interfacedoc def post_process(self): parent_result = self.process_pipe.results[self._result_id] self.image = parent_result._render_PIL( (self.image_width, self.image_height), self.dpi) @classmethod def create(cls, analyzer, result_id, grapher_id, grapher_name): class NewGrapher(cls): _id = grapher_id implements(IGrapher) @interfacedoc def __init__(self, width=1024, height=256, bg_color=(0, 0, 0), color_scheme='default'): super(NewGrapher, self).__init__(width, height, bg_color, color_scheme) self.parents.append(analyzer) self._result_id = result_id # TODO : make it generic when analyzer will be "atomize" @staticmethod @interfacedoc def id(): return grapher_id @staticmethod @interfacedoc def name(): return grapher_name __doc__ = """Builds a PIL image representing """ + grapher_name NewGrapher.__name__ = 'Display' + result_id return NewGrapher