def setup(self): """ Generic setup for any analysis module, can be overriden by implementing in any child class This sets up subscriber and publisher based on input and output feature names """ # usually this module is used with incoming EEG, # so we'd like to know num channels, and a header is for convenience # hard-coded "eeg" could be a problem if the device's metric name for raw data is not "eeg" # currently "eeg" is a known good metric name for both OpenBCI and Muse if "num_channels" in self.module_settings: # module specific setting self.num_channels = self.module_settings["num_channels"] elif "num_channels" in self.global_settings: #global setting self.num_channels = self.global_settings["num_channels"] else: # default for the device type self.num_channels = get_num_channels(self.device_name,"eeg") self.headers = ['timestamp'] + ['channel_%s' % i for i in xrange(self.num_channels)] # if input, instantiate subscriber if len(self.inputs): # there is only one subscriber to handle all inputs self.subscriber = PikaSubscriber(device_name=self.device_name, device_id=self.device_id, rabbitmq_address=self.rabbitmq_address, metrics=self.inputs) # if output, instantiate publishers if len(self.outputs): for output_key, output in self.outputs.iteritems(): # each output has a specific key, assign a placeholder for it in publishers collection self.publishers[output_key] = {} self.output_buffers[output_key] = {} # each output has a parameter called "message_queues" # this can be a single value or a list, i.e. "foo" or ["foo1","foo2"] # most of the time, this will be a single string # an example of where an output might use more than one message_queue might be: # one output goes to visualization, while a second copy continues down the processing chain if 'message_queues' in output: # for convenience, convert the "message_queues" parameter to list if it isn't already if type(output['message_queues']) != list: output['message_queues'] = [output['message_queues']] # there is one publisher per output for message_queue_name in output['message_queues']: self.publishers[output_key][message_queue_name] = PikaPublisher( device_name=self.device_name, device_id=self.device_id, rabbitmq_address=self.rabbitmq_address, metric_name=message_queue_name) # also instantiate an output buffer for each publisher self.output_buffers[output_key][message_queue_name] = []
class ModuleAbstract(object): __metaclass__ = ABCMeta MODULE_NAME = "Abstract" # simple JSON message and corresponding data types MESSAGE_TYPE_JSON = "JSON" # base64 encoded matrix messages and corresponding data types MESSAGE_TYPE_MATRIX = "MATRIX" # data types DATA_TYPE_CLASS_LABELS = "CLASS_LABELS" DATA_TYPE_RAW_DATA = "RAW_DATA" DATA_TYPE_RAW_POINTS = "RAW_POINTS" DATA_TYPE_LABELED_DATA = "LABELED_DATA" DATA_TYPE_LABELED_POINTS = "LABELED_POINTS" def __init__(self, device_name, device_id, rabbitmq_address, module_conf, global_conf): """ global constructor for all module classes, not meant to be overwritten by subclasses :param device_name: :param device_id: :param rabbitmq_address: :param moduleConf: :return: """ # set global properties common to all self.device_name = device_name self.device_id = device_id self.rabbitmq_address = rabbitmq_address self.module_conf = module_conf self.global_conf = global_conf # id self.id = None if 'id' in self.module_conf: self.id = self.module_conf['id'] # inputs (optional) self.inputs = {} if 'inputs' in self.module_conf: self.inputs = self.module_conf['inputs'] # outputs (optional) self.outputs = {} if 'outputs' in self.module_conf: self.outputs = self.module_conf['outputs'] # module parameters (optional) self.module_params = None if 'parameters' in self.module_conf: self.module_params = self.module_conf['parameters'] self.subscriber = None self.publishers = {} self.output_buffers = {} self.num_channels = 0 self.headers = [] # debug self.debug = False if 'debug' in self.module_params: if self.module_params['debug'] is True: self.debug = True # if len(self.inputs): # print "inputs:" + str(self.inputs) # if len(self.outputs): # print "outputs:" + str(self.outputs) # call setup() self.setup() def setup(self): """ Generic setup for any analysis module, can be overriden by implementing in any child class This sets up subscriber and publisher based on input and output feature names """ # usually this module is used with incoming EEG, # so we'd like to know num channels, and a header is for convenience # hard-coded "eeg" could be a problem if the device's metric name for raw data is not "eeg" self.num_channels = get_num_channels(self.device_name,"eeg") self.headers = ['timestamp'] + ['channel_%s' % i for i in xrange(self.num_channels)] # if input, instantiate subscriber if len(self.inputs): # there is only one subscriber to handle all inputs self.subscriber = PikaSubscriber(device_name=self.device_name, device_id=self.device_id, rabbitmq_address=self.rabbitmq_address, metrics=self.inputs) # if output, instantiate publishers if len(self.outputs): for output_key, output in self.outputs.iteritems(): # each output has a specific key, assign a placeholder for it in publishers collection self.publishers[output_key] = {} self.output_buffers[output_key] = {} # each output has a parameter called "message_queues" # this can be a single value or a list, i.e. "foo" or ["foo1","foo2"] # most of the time, this will be a single string # an example of where an output might use more than one message_queue might be: # one output goes to visualization, while a second copy continues down the processing chain # for convenience, convert the "message_queues" parameter to list if it isn't already if type(output['message_queues']) != list: output['message_queues'] = [output['message_queues']] # there is one publisher per output for message_queue_name in output['message_queues']: self.publishers[output_key][message_queue_name] = PikaPublisher( device_name=self.device_name, device_id=self.device_id, rabbitmq_address=self.rabbitmq_address, metric_name=message_queue_name) # also instantiate an output buffer for each publisher self.output_buffers[output_key][message_queue_name] = [] def start(self): """ Consume and write data to file :return: """ # unleash the hounds! if len(self.publishers): if self.debug: print "[" + self.MODULE_NAME + "] starting publishers" for output_key, output_message_queues in self.publishers.iteritems(): for output_message_queue, publisher in output_message_queues.iteritems(): publisher.connect() if self.debug: print "[" + self.MODULE_NAME + "] publisher [" + output_key + "][" + output_message_queue + "] started streaming" if self.subscriber is None: # if there are publishers but no subscribers defined, this is a generator type module # so we need some way to start publishing messages without depending on the consume() method self.generate() # it begins! if self.subscriber and self.debug: print "[" + self.MODULE_NAME + "] starting subscribers" self.subscriber.connect() self.subscriber.consume_messages(self.consume) return def stop(self): """ Unsubscribe and close file :return: """ print "Abstract: stopped" self.subscriber.disconnect() def generate(self): """ generate to the message queue :return: """ print "Abstract: generate" def consume(self, ch, method, properties, body): """ consume the message queue from rabbitmq :return: """ print "Abstract: consume" def write(self, output_key, datum): """ deliver one message (buffered) """ for message_queue_name in self.output_buffers[output_key].keys(): # add one data point to each buffer defined for this output self.output_buffers[output_key][message_queue_name].append(datum) # if buffer is full, publish it, using appropriate publisher if len(self.output_buffers[output_key][message_queue_name]) >= self.outputs[output_key]['buffer_size']: # publishing means deliver the buffer (which is now full) to the publisher # this sends the entire buffer onto message queue self.publishers[output_key][message_queue_name].publish(self.output_buffers[output_key][message_queue_name]) # and then reset the buffer self.output_buffers[output_key][message_queue_name] = []
class ModuleAbstract(object): __metaclass__ = ABCMeta MODULE_NAME = "Abstract" def __init__(self, device_name, device_id, rabbitmq_address, module_conf={}, global_conf={}, module_index=0): """ global constructor for all module classes, not meant to be overwritten by subclasses """ self.color = colors.CYCLE[module_index % 10] # LOGNAME is a prefix used to prepaend to debugging output statements, helps to disambiguate messages since the # modules run on separate threads self.LOGNAME = self.color + "[Module: " + self.MODULE_NAME + "] " + colors.ENDC # set global properties common to all self.device_name = device_name self.device_id = device_id self.rabbitmq_address = rabbitmq_address self.module_conf = module_conf self.global_conf = global_conf self.global_settings = self.global_conf["global"] if self.global_conf and "global" in self.global_conf else {} # id self.id = None if 'id' in self.module_conf: self.id = self.module_conf['id'] # inputs (optional) self.inputs = {} if 'inputs' in self.module_conf: self.inputs = self.module_conf['inputs'] # outputs (optional) self.outputs = {} if 'outputs' in self.module_conf: self.outputs = self.module_conf['outputs'] # module parameters (optional) self.module_settings = None if 'settings' in self.module_conf: self.module_settings = self.module_conf['settings'] self.subscriber = None self.publishers = {} self.output_buffers = {} self.num_channels = 0 self.headers = [] # debug self.debug = False if 'debug' in self.module_settings: if self.module_settings['debug'] is True: self.debug = True # if len(self.inputs): # print "inputs:" + str(self.inputs) # if len(self.outputs): # print "outputs:" + str(self.outputs) # call setup() self.setup() def extractSettings(self, all_settings, settings_to_extract): for key, default_value in settings_to_extract.iteritems(): #print "setting", key self.__dict__[key] = all_settings[key] if key in all_settings else default_value def setup(self): """ Generic setup for any analysis module, can be overriden by implementing in any child class This sets up subscriber and publisher based on input and output feature names """ # usually this module is used with incoming EEG, # so we'd like to know num channels, and a header is for convenience # hard-coded "eeg" could be a problem if the device's metric name for raw data is not "eeg" # currently "eeg" is a known good metric name for both OpenBCI and Muse if "num_channels" in self.module_settings: # module specific setting self.num_channels = self.module_settings["num_channels"] elif "num_channels" in self.global_settings: #global setting self.num_channels = self.global_settings["num_channels"] else: # default for the device type self.num_channels = get_num_channels(self.device_name,"eeg") self.headers = ['timestamp'] + ['channel_%s' % i for i in xrange(self.num_channels)] # if input, instantiate subscriber if len(self.inputs): # there is only one subscriber to handle all inputs self.subscriber = PikaSubscriber(device_name=self.device_name, device_id=self.device_id, rabbitmq_address=self.rabbitmq_address, metrics=self.inputs) # if output, instantiate publishers if len(self.outputs): for output_key, output in self.outputs.iteritems(): # each output has a specific key, assign a placeholder for it in publishers collection self.publishers[output_key] = {} self.output_buffers[output_key] = {} # each output has a parameter called "message_queues" # this can be a single value or a list, i.e. "foo" or ["foo1","foo2"] # most of the time, this will be a single string # an example of where an output might use more than one message_queue might be: # one output goes to visualization, while a second copy continues down the processing chain if 'message_queues' in output: # for convenience, convert the "message_queues" parameter to list if it isn't already if type(output['message_queues']) != list: output['message_queues'] = [output['message_queues']] # there is one publisher per output for message_queue_name in output['message_queues']: self.publishers[output_key][message_queue_name] = PikaPublisher( device_name=self.device_name, device_id=self.device_id, rabbitmq_address=self.rabbitmq_address, metric_name=message_queue_name) # also instantiate an output buffer for each publisher self.output_buffers[output_key][message_queue_name] = [] def start(self): """ Consume and write data to file :return: """ # unleash the hounds! if len(self.publishers): if self.debug: print self.LOGNAME + " starting publishers" for output_key, output_message_queues in self.publishers.iteritems(): for output_message_queue, publisher in output_message_queues.iteritems(): publisher.connect() #if self.debug: # print self.LOGNAME + " publisher [" + output_key + "][" + output_message_queue + "] started streaming" print self.LOGNAME + " publisher [" + self.device_id + ":" + self.device_name + ":" + output_message_queue + "] started streaming" if self.subscriber is None: # if there are publishers but no subscribers defined, this is a generator type module # so we need some way to start publishing messages without depending on the consume() method self.generate() # it begins! if self.subscriber and self.debug: print self.LOGNAME + " starting subscribers" self.subscriber.connect() self.subscriber.consume_messages(self.consume) return def stop(self): """ Unsubscribe and close file :return: """ print "Abstract: stopped" self.subscriber.disconnect() def generate(self): """ generate to the message queue :return: """ print "Abstract: generate" def consume(self, ch, method, properties, body): """ consume the message queue from rabbitmq :return: """ print "Abstract: consume" def write(self, output_key, datum): """ deliver one message (buffered) """ for message_queue_name in self.output_buffers[output_key].keys(): # add one data point to each buffer defined for this output self.output_buffers[output_key][message_queue_name].append(datum) # if buffer is full, publish it, using appropriate publisher if len(self.output_buffers[output_key][message_queue_name]) >= self.outputs[output_key]['buffer_size']: # publishing means deliver the buffer (which is now full) to the publisher # this sends the entire buffer onto message queue self.publishers[output_key][message_queue_name].publish(self.output_buffers[output_key][message_queue_name]) # and then reset the buffer self.output_buffers[output_key][message_queue_name] = []
class ModuleAbstract(object): __metaclass__ = ABCMeta MODULE_NAME = "Abstract" def __init__(self, device_name, device_id, rabbitmq_address, module_conf={}, global_conf={}, module_index=0): """ global constructor for all module classes, not meant to be overwritten by subclasses """ self.color = colors.CYCLE[module_index % 10] # LOGNAME is a prefix used to prepaend to debugging output statements, helps to disambiguate messages since the # modules run on separate threads self.LOGNAME = self.color + "[Module: " + self.MODULE_NAME + "] " + colors.ENDC # set global properties common to all self.device_name = device_name self.device_id = device_id self.rabbitmq_address = rabbitmq_address self.module_conf = module_conf self.global_conf = global_conf self.global_settings = self.global_conf["global"] if self.global_conf and "global" in self.global_conf else {} # id self.id = None if 'id' in self.module_conf: self.id = self.module_conf['id'] # inputs (optional) self.inputs = {} if 'inputs' in self.module_conf: self.inputs = self.module_conf['inputs'] # outputs (optional) self.outputs = {} if 'outputs' in self.module_conf: self.outputs = self.module_conf['outputs'] # module parameters (optional) self.module_settings = None if 'settings' in self.module_conf: self.module_settings = self.module_conf['settings'] self.subscriber = None self.publishers = {} self.output_buffers = {} self.num_channels = 0 self.headers = [] # debug self.debug = False if 'debug' in self.module_settings: if self.module_settings['debug'] is True: self.debug = True # if len(self.inputs): # print "inputs:" + str(self.inputs) # if len(self.outputs): # print "outputs:" + str(self.outputs) # call setup() self.setup() def extractSettings(self, all_settings, settings_to_extract): for key, default_value in settings_to_extract.iteritems(): #print "setting", key self.__dict__[key] = all_settings[ key] if key in all_settings else default_value def setup(self): """ Generic setup for any analysis module, can be overriden by implementing in any child class This sets up subscriber and publisher based on input and output feature names """ # usually this module is used with incoming EEG, # so we'd like to know num channels, and a header is for convenience # hard-coded "eeg" could be a problem if the device's metric name for raw data is not "eeg" # currently "eeg" is a known good metric name for both OpenBCI and Muse if "num_channels" in self.module_settings: # module specific setting self.num_channels = self.module_settings["num_channels"] elif "num_channels" in self.global_settings: #global setting self.num_channels = self.global_settings["num_channels"] else: # default for the device type self.num_channels = get_num_channels(self.device_name, "eeg") self.headers = ['timestamp'] + [ 'channel_%s' % i for i in xrange(self.num_channels) ] # if input, instantiate subscriber if len(self.inputs): # there is only one subscriber to handle all inputs self.subscriber = PikaSubscriber( device_name=self.device_name, device_id=self.device_id, rabbitmq_address=self.rabbitmq_address, metrics=self.inputs) # if output, instantiate publishers if len(self.outputs): for output_key, output in self.outputs.iteritems(): # each output has a specific key, assign a placeholder for it in publishers collection self.publishers[output_key] = {} self.output_buffers[output_key] = {} # each output has a parameter called "message_queues" # this can be a single value or a list, i.e. "foo" or ["foo1","foo2"] # most of the time, this will be a single string # an example of where an output might use more than one message_queue might be: # one output goes to visualization, while a second copy continues down the processing chain if 'message_queues' in output: # for convenience, convert the "message_queues" parameter to list if it isn't already if type(output['message_queues']) != list: output['message_queues'] = [output['message_queues']] # there is one publisher per output for message_queue_name in output['message_queues']: self.publishers[output_key][ message_queue_name] = PikaPublisher( device_name=self.device_name, device_id=self.device_id, rabbitmq_address=self.rabbitmq_address, metric_name=message_queue_name) # also instantiate an output buffer for each publisher self.output_buffers[output_key][ message_queue_name] = [] def start(self): """ Consume and write data to file :return: """ # unleash the hounds! if len(self.publishers): if self.debug: print self.LOGNAME + " starting publishers" for output_key, output_message_queues in self.publishers.iteritems( ): for output_message_queue, publisher in output_message_queues.iteritems( ): publisher.connect() #if self.debug: # print self.LOGNAME + " publisher [" + output_key + "][" + output_message_queue + "] started streaming" print self.LOGNAME + " publisher [" + self.device_id + ":" + self.device_name + ":" + output_message_queue + "] started streaming" if self.subscriber is None: # if there are publishers but no subscribers defined, this is a generator type module # so we need some way to start publishing messages without depending on the consume() method self.generate() # it begins! if self.subscriber: if self.debug: print self.LOGNAME + " starting subscribers" self.subscriber.connect() self.subscriber.consume_messages(self.consume) return def stop(self): """ Unsubscribe and close file :return: """ print "Abstract: stopped" self.subscriber.disconnect() def generate(self): """ generate to the message queue :return: """ print "Abstract: generate" def consume(self, ch, method, properties, body): """ consume the message queue from rabbitmq :return: """ print "Abstract: consume" def write(self, output_key, datum): """ deliver one message (buffered) """ for message_queue_name in self.output_buffers[output_key].keys(): # add one data point to each buffer defined for this output self.output_buffers[output_key][message_queue_name].append(datum) # if buffer is full, publish it, using appropriate publisher if len(self.output_buffers[output_key][message_queue_name] ) >= self.outputs[output_key]['buffer_size']: # publishing means deliver the buffer (which is now full) to the publisher # this sends the entire buffer onto message queue self.publishers[output_key][message_queue_name].publish( self.output_buffers[output_key][message_queue_name]) # and then reset the buffer self.output_buffers[output_key][message_queue_name] = []