def helper_get_probe_data(self, probe): ret = [] records = 1 storage = Storage(self.config["datafile"]) for r in storage.get_records_by_probe_id(probe.id, records): record = { "metadata": { "probe_id": r[0], "timestamp": r[2], "length": r[3], }, "data": str(r[4]).encode('utf-8') } if (record["data"].startswith('{') or record["data"].startswith('[')): try: record["data"] = json.loads(record["data"]) if type(record["data"]) == dict: record["data"] = [record["data"]] except: pass ret.append(record) storage.close() return ret
def main(): writer = csv.writer(sys.stdout) storage = Storage(config['datafile']) for r in storage.reader(): if r[0] == 0x40: # camera fname = "%s-%s.jpg" % (r[0], r[1]) with open(fname, 'w') as f: f.write(r[3]) print "wrote: %s" % fname else: writer.writerow(r)
def main(): writer = csv.writer(sys.stdout) storage = Storage(config['datafile']) for r in storage.reader(): if r[0] == 0x40: # camera fname ="%s-%s.jpg" % (r[0], r[1]) with open(fname, 'w') as f: f.write(r[3]) print "wrote: %s" % fname else: writer.writerow(r)
def __init__(self, datafile, probespath, probe_confs): self.active = False self.active_probes = 0 self.probespath = probespath self.loop = pyev.Loop() #self.queue = Queue() # initialize and start a idle watcher #self.idle_watcher = pyev.Idle(self.loop, self.idle_cb) #self.idle_watcher.start() # initialize and start a signal watchers sigterm_watcher = pyev.Signal(signal.SIGTERM, self.loop, self.sigterm_cb) sigterm_watcher.start() sigint_watcher = pyev.Signal(signal.SIGINT, self.loop, self.sigint_cb) sigint_watcher.start() self.loop.data = [sigterm_watcher, sigint_watcher] #self.loop.data.append(self.idle_watcher) # initialize storage logging.info("Init storage...") self.storage = Storage(datafile) # read in probes from config self.probes = [] index = 0 for probe_conf in probe_confs: self.check_command(probe_conf) self.check_data_type(probe_conf) self.check_dummy(probe_conf) # load filters self.load_filters(probe_conf) # load error filters self.load_error_filters(probe_conf) p = self.get_probe(index, self.storage, probe_conf, -1) p.register(self.loop) self.probes.append(p) if probe_conf.get("auto_start", False): self.start_probe(index) index = index + 1
def open_storage(self): try: self.storage = Storage(self.config['datafile']) logging.info("Storage opened") except Exception as ex: logging.error("Could not open Storage: %s. Exiting." % ex) self.close() sys.exit(5)
def main(): storage = Storage(config['datafile']) for r in storage.get_records_by_probe_id('meerkat.probe.camera_photo', 3): data = str(r[4]).encode('utf-8') print r[0], r[1], r[2], r[3], data
class Scheduler(object): def __init__(self, datafile, probespath, probe_confs): self.active = False self.active_probes = 0 self.probespath = probespath self.loop = pyev.Loop() #self.queue = Queue() # initialize and start a idle watcher #self.idle_watcher = pyev.Idle(self.loop, self.idle_cb) #self.idle_watcher.start() # initialize and start a signal watchers sigterm_watcher = pyev.Signal(signal.SIGTERM, self.loop, self.sigterm_cb) sigterm_watcher.start() sigint_watcher = pyev.Signal(signal.SIGINT, self.loop, self.sigint_cb) sigint_watcher.start() self.loop.data = [sigterm_watcher, sigint_watcher] #self.loop.data.append(self.idle_watcher) # initialize storage logging.info("Init storage...") self.storage = Storage(datafile) # read in probes from config self.probes = [] index = 0 for probe_conf in probe_confs: self.check_command(probe_conf) self.check_data_type(probe_conf) self.check_dummy(probe_conf) # load filters self.load_filters(probe_conf) # load error filters self.load_error_filters(probe_conf) p = self.get_probe(index, self.storage, probe_conf, -1) p.register(self.loop) self.probes.append(p) if probe_conf.get("auto_start", False): self.start_probe(index) index = index + 1 ''' def idle_cb(self, watcher, revents): #self.idle_watcher.stop() try: command = self.queue.get() except Empty: command = None if command: if command[0] == COMMAND_EXEC: logging.debug("Idle: exec command: %s" % (command,)) # pass the rest of the command elements as args to method getattr(self, command[1])(*command[2:]) else: logging.debug("Idle: got unknown command: %s. Ignoring." % command) #self.idle_watcher.start() ''' def sigterm_cb(self, watcher, revents): logging.info("SIGTERM caught. Exiting..") self.halt() def sigint_cb(self, watcher, revents): logging.info("SIGINT caught. Exiting..") self.halt() def start(self, paused=True): if not paused: self.start_probes() logging.info("Event loop start") self.loop.start() def start_probes(self): logging.info("Start all probes") for p in xrange(len(self.probes)): self.start_probe(p) def start_probe(self, p): logging.info("Start probe: %s" % self.probes[p].id) if not self.probes[p].running: self.active_probes = self.active_probes + 1 self.probes[p].start() self.active = True def stop_probes(self): logging.info("Stop all probes") if self.probes: for p in xrange(len(self.probes)): self.stop_probe(p) def stop_probe(self, p): logging.info("Stop probe: %s" % self.probes[p].id) if self.probes[p].running: self.active_probes = self.active_probes - 1 self.probes[p].stop() if self.active_probes == 0: self.active = False def halt(self): logging.info("Halting...") if self.loop.data: while self.loop.data: self.loop.data.pop().stop() if self.probes: self.stop_probes() logging.info("Closing storage...") self.storage.close() self.loop.stop(pyev.EVBREAK_ALL) def load_filters(self, probe_conf): self._load_filters(probe_conf, "filters") def load_error_filters(self, probe_conf): self._load_filters(probe_conf, "error_filters") def _load_filters(self, probe_conf, filter_conf_key): if not filter_conf_key in probe_conf: probe_conf[filter_conf_key] = [] return # Dynamically load filters for then probe. # Replace the module/class name with an actual instance. for i, module in enumerate(probe_conf[filter_conf_key]): filter_id = module try: parts = module.split(".") cls = parts[-1] module = ".".join(parts[:-1]) __import__(module, locals(), globals()) except: raise MeerkatException("Could not import filter module: %s" % module) if sys.modules.has_key(module): if hasattr(sys.modules[module], cls): probe_conf[filter_conf_key][i] = getattr(sys.modules[module], cls)(filter_id) else: raise MeerkatException("Module %s has no class %s" % (module, cls)) else: raise MeerkatException("Could not load filter module: %s" % module) def check_dummy(self, probe_conf): # ensure dummy param exists (default False) if not "dummy" in probe_conf: probe_conf["dummy"] = False def check_data_type(self, probe_conf): # sanity check the probe data type if not "data_type" in probe_conf \ or not probe_conf["data_type"] in [probe.DATA_TYPE_TEXT, probe.DATA_TYPE_JSON, probe.DATA_TYPE_DATA]: raise ValueError("Bad config: %s: missing or invalid 'data_type' attribute" % probe_conf["id"]) def check_command(self, probe_conf): # sanity check the probe command if not "command" in probe_conf or len(probe_conf["command"]) == 0: raise ValueError("Bad config: %s: missing or empty 'command' attribute" % probe_conf["id"]) if not type(probe_conf["command"]) == type([]): raise ValueError("Bad config: %s: 'command' should be an array of strings" % probe_conf["id"]) # expand the command, saves doing this each time probe_conf["command"][0] = os.path.join(self.probespath, probe_conf["command"][0]) def get_probe(self, index, storage, probe_conf, timeout): # [FIXME: this is a bit dense] if not "type" in probe_conf: raise ValueError("Bad config: %s does not have a 'type' attribute" % probe_conf["id"]) if probe_conf["type"] == probe.TYPE_DURATION: if not "interval" in probe_conf or not "duration" in probe_conf: raise ValueError("Bad config: %s: probes of this type must have a 'interval' attribute \ and a 'duration' attribute" % probe_conf["id"]) return probe.Probe(probe_conf["id"], index, storage, probe_conf, timeout) elif probe_conf["type"] == probe.TYPE_PERIODIC: if not "interval" in probe_conf: raise ValueError("Bad config: %s: probes of this type must have a 'interval' attribute" % probe_conf["id"]) probe_conf["duration"] = -1 return probe.Probe(probe_conf["id"], index, storage, probe_conf, timeout) elif probe_conf["type"] == probe.TYPE_CONTINUOUS: raise NotImplementedError("Probe type not yet implemented: %s" % probe_conf["type"]) else: raise NotImplementedError("No such probe type: %s" % probe_conf["type"])
class Scheduler(object): def __init__(self, datafile, probespath, probe_confs): self.active = False self.active_probes = 0 self.probespath = probespath self.loop = pyev.Loop() #self.queue = Queue() # initialize and start a idle watcher #self.idle_watcher = pyev.Idle(self.loop, self.idle_cb) #self.idle_watcher.start() # initialize and start a signal watchers sigterm_watcher = pyev.Signal(signal.SIGTERM, self.loop, self.sigterm_cb) sigterm_watcher.start() sigint_watcher = pyev.Signal(signal.SIGINT, self.loop, self.sigint_cb) sigint_watcher.start() self.loop.data = [sigterm_watcher, sigint_watcher] #self.loop.data.append(self.idle_watcher) # initialize storage logging.info("Init storage...") self.storage = Storage(datafile) # read in probes from config self.probes = [] index = 0 for probe_conf in probe_confs: self.check_command(probe_conf) self.check_data_type(probe_conf) self.check_dummy(probe_conf) # load filters self.load_filters(probe_conf) # load error filters self.load_error_filters(probe_conf) p = self.get_probe(index, self.storage, probe_conf, -1) p.register(self.loop) self.probes.append(p) if probe_conf.get("auto_start", False): self.start_probe(index) index = index + 1 ''' def idle_cb(self, watcher, revents): #self.idle_watcher.stop() try: command = self.queue.get() except Empty: command = None if command: if command[0] == COMMAND_EXEC: logging.debug("Idle: exec command: %s" % (command,)) # pass the rest of the command elements as args to method getattr(self, command[1])(*command[2:]) else: logging.debug("Idle: got unknown command: %s. Ignoring." % command) #self.idle_watcher.start() ''' def sigterm_cb(self, watcher, revents): logging.info("SIGTERM caught. Exiting..") self.halt() def sigint_cb(self, watcher, revents): logging.info("SIGINT caught. Exiting..") self.halt() def start(self, paused=True): if not paused: self.start_probes() logging.info("Event loop start") self.loop.start() def start_probes(self): logging.info("Start all probes") for p in xrange(len(self.probes)): self.start_probe(p) def start_probe(self, p): logging.info("Start probe: %s" % self.probes[p].id) if not self.probes[p].running: self.active_probes = self.active_probes + 1 self.probes[p].start() self.active = True def stop_probes(self): logging.info("Stop all probes") if self.probes: for p in xrange(len(self.probes)): self.stop_probe(p) def stop_probe(self, p): logging.info("Stop probe: %s" % self.probes[p].id) if self.probes[p].running: self.active_probes = self.active_probes - 1 self.probes[p].stop() if self.active_probes == 0: self.active = False def halt(self): logging.info("Halting...") if self.loop.data: while self.loop.data: self.loop.data.pop().stop() if self.probes: self.stop_probes() logging.info("Closing storage...") self.storage.close() self.loop.stop(pyev.EVBREAK_ALL) def load_filters(self, probe_conf): self._load_filters(probe_conf, "filters") def load_error_filters(self, probe_conf): self._load_filters(probe_conf, "error_filters") def _load_filters(self, probe_conf, filter_conf_key): if not filter_conf_key in probe_conf: probe_conf[filter_conf_key] = [] return # Dynamically load filters for then probe. # Replace the module/class name with an actual instance. for i, module in enumerate(probe_conf[filter_conf_key]): filter_id = module try: parts = module.split(".") cls = parts[-1] module = ".".join(parts[:-1]) __import__(module, locals(), globals()) except: raise MeerkatException("Could not import filter module: %s" % module) if sys.modules.has_key(module): if hasattr(sys.modules[module], cls): probe_conf[filter_conf_key][i] = getattr( sys.modules[module], cls)(filter_id) else: raise MeerkatException("Module %s has no class %s" % (module, cls)) else: raise MeerkatException("Could not load filter module: %s" % module) def check_dummy(self, probe_conf): # ensure dummy param exists (default False) if not "dummy" in probe_conf: probe_conf["dummy"] = False def check_data_type(self, probe_conf): # sanity check the probe data type if not "data_type" in probe_conf \ or not probe_conf["data_type"] in [probe.DATA_TYPE_TEXT, probe.DATA_TYPE_JSON, probe.DATA_TYPE_DATA]: raise ValueError( "Bad config: %s: missing or invalid 'data_type' attribute" % probe_conf["id"]) def check_command(self, probe_conf): # sanity check the probe command if not "command" in probe_conf or len(probe_conf["command"]) == 0: raise ValueError( "Bad config: %s: missing or empty 'command' attribute" % probe_conf["id"]) if not type(probe_conf["command"]) == type([]): raise ValueError( "Bad config: %s: 'command' should be an array of strings" % probe_conf["id"]) # expand the command, saves doing this each time probe_conf["command"][0] = os.path.join(self.probespath, probe_conf["command"][0]) def get_probe(self, index, storage, probe_conf, timeout): # [FIXME: this is a bit dense] if not "type" in probe_conf: raise ValueError( "Bad config: %s does not have a 'type' attribute" % probe_conf["id"]) if probe_conf["type"] == probe.TYPE_DURATION: if not "interval" in probe_conf or not "duration" in probe_conf: raise ValueError( "Bad config: %s: probes of this type must have a 'interval' attribute \ and a 'duration' attribute" % probe_conf["id"]) return probe.Probe(probe_conf["id"], index, storage, probe_conf, timeout) elif probe_conf["type"] == probe.TYPE_PERIODIC: if not "interval" in probe_conf: raise ValueError( "Bad config: %s: probes of this type must have a 'interval' attribute" % probe_conf["id"]) probe_conf["duration"] = -1 return probe.Probe(probe_conf["id"], index, storage, probe_conf, timeout) elif probe_conf["type"] == probe.TYPE_CONTINUOUS: raise NotImplementedError("Probe type not yet implemented: %s" % probe_conf["type"]) else: raise NotImplementedError("No such probe type: %s" % probe_conf["type"])