def get_processable(self, agent_id, desc_domain, selector): if not format_check.is_valid_domain(desc_domain): return [] if not format_check.is_valid_fullselector(selector): return [] log.debug("GET_PROCESSABLE: %s:%s %s", desc_domain, selector, agent_id) return self.store.get_processable(str(desc_domain), str(selector))
def list_uuids(self, agent_id, desc_domain): log.debug("LISTUUIDS: %s %s", agent_id, desc_domain) if not self._check_agent_id(agent_id): return {} if not format_check.is_valid_domain(desc_domain): return {} return self.store.list_uuids(str(desc_domain))
def processed_stats(self, agent_id, desc_domain): log.debug("PROCESSED_STATS: %s %s", agent_id, desc_domain) if not self._check_agent_id(agent_id): return [] if not format_check.is_valid_domain(desc_domain): return [] return self.store.processed_stats(str(desc_domain))
def unlock(self, agent_id, lockid, desc_domain, selector, processing_failed, retries, wait_time): if not format_check.is_valid_domain(desc_domain): return if not format_check.is_valid_fullselector(selector): return objpath = self.clients[agent_id] locks = self.locks[desc_domain] lkey = (lockid, selector) log.debug("UNLOCK:%s %s(%s) => %r %d:%d ", lockid, objpath, agent_id, processing_failed, retries, wait_time) if lkey not in locks: return locks.remove(lkey) # find agent_name, config_txt for (agent_name, config_txt), ids in self.uniq_conf_clients.items(): if agent_id in ids: break rkey = (agent_name, config_txt, desc_domain, selector) if rkey not in self.retry_counters: self.retry_counters[rkey] = retries if self.retry_counters[rkey] > 0: self.retry_counters[rkey] -= 1 desc = self.store.get_descriptor(desc_domain, selector) uuid = desc.uuid self.sched.add_action( wait_time, (agent_id, desc_domain, uuid, selector, agent_name))
def register(self, agent_id, agent_domain, pth, config_txt, processes_descriptors): #: indicates whether another instance of the same agent is already #: running with the same configuration if not format_check.is_valid_domain(agent_domain): return agent_name = agent_id.split('-', 1)[0] self.agentnames[agent_id] = agent_name output_altering_options = get_output_altering_options(str(config_txt)) name_config = (agent_name, output_altering_options) already_running = len(self.uniq_conf_clients[name_config]) > 1 self.uniq_conf_clients[name_config].append(agent_id) self.clients[agent_id] = pth self.agents_output_altering_options[agent_id] = output_altering_options self.agents_full_config_txts[agent_id] = str(config_txt) log.info("New client %s (%s) in domain %s with config %s", pth, agent_id, agent_domain, config_txt) if not processes_descriptors: self.descriptor_handled_count[name_config] = 0 elif not already_running: # Send not-yet processed descriptors to the agent, # unless another instance of the same agent has already been # started, and should be processing those descriptors unprocessed = \ self.store.list_unprocessed_by_agent(agent_name, output_altering_options) self.descriptor_handled_count[name_config] = \ self.descriptor_count - len(unprocessed) for dom, uuid, sel in unprocessed: self.targeted_descriptor("storage", dom, uuid, sel, [agent_name], False)
def find_by_uuid(self, agent_id, desc_domain, uuid): log.debug("FINDBYUUID: %s %s:%s", agent_id, desc_domain, uuid) if not self._check_agent_id(agent_id): return [] if not format_check.is_valid_domain(desc_domain): return [] descs = self.store.find_by_uuid(str(desc_domain), str(uuid)) return [desc.serialize_meta(serializer) for desc in descs]
def find(self, agent_id, desc_domain, selector_regex, limit=0, offset=0): log.debug("FIND: %s %s:%s (max %d skip %d)", agent_id, desc_domain, selector_regex, limit, offset) if not format_check.is_valid_domain(desc_domain): return [] selectors = self.store.find(str(desc_domain), str(selector_regex), str(limit), int(offset)) return [str(s) for s in selectors]
def get_children(self, agent_id, desc_domain, selector, recurse): log.debug("GET_CHILDREN: %s %s:%s", agent_id, desc_domain, selector) if not format_check.is_valid_domain(desc_domain): return [] if not format_check.is_valid_fullselector(selector): return [] descs = self.store.get_children(str(desc_domain), str(selector), recurse=bool(recurse)) return [desc.serialize_meta(serializer) for desc in descs]
def find_by_value(self, agent_id, desc_domain, selector_prefix, value_regex): log.debug("FINDBYVALUE: %s %s %s %s", agent_id, desc_domain, selector_prefix, value_regex) if not format_check.is_valid_domain(desc_domain): return [] descs = self.store.find_by_value(str(desc_domain), str(selector_prefix), str(value_regex)) return [desc.serialize_meta(serializer) for desc in descs]
def get_value(self, agent_id, desc_domain, selector): log.debug("GETVALUE: %s %s:%s", agent_id, desc_domain, selector) if not format_check.is_valid_domain(desc_domain): return None if not format_check.is_valid_selector(selector): return None value = self.store.get_value(str(desc_domain), str(selector)) if value is None: return "" return serializer.dumps(value)
def get(self, agent_id, desc_domain, selector): log.debug("GET: %s %s:%s", agent_id, desc_domain, selector) if not format_check.is_valid_domain(desc_domain): return None if not format_check.is_valid_selector(selector): return None desc = self.store.get_descriptor(str(desc_domain), str(selector)) if desc is None: return "" return desc.serialize_meta(serializer)
def request_processing(self, agent_id, desc_domain, selector, targets): log.debug("REQUEST_PROCESSING: %s %s:%s targets %s", agent_id, desc_domain, selector, [str(t) for t in targets]) if not format_check.is_valid_domain(desc_domain): return if not format_check.is_valid_fullselector(selector): return d = self.store.get_descriptor(str(desc_domain), str(selector)) self.userrequestid += 1 self.targeted_descriptor(agent_id, desc_domain, d.uuid, selector, targets, self.userrequestid)
def mark_processable(self, agent_id, desc_domain, selector): if not format_check.is_valid_domain(desc_domain): return if not format_check.is_valid_fullselector(selector): return agent_name = self.agentnames[agent_id] options = self.agents_output_altering_options[agent_id] log.debug("MARK_PROCESSABLE: %s:%s %s %s", desc_domain, selector, agent_id, options) isnew = self.store.mark_processable(str(desc_domain), str(selector), agent_name, str(options)) if isnew: self._update_check_idle(agent_name, options)
def get_children(self, agent_id, desc_domain, selector, recurse): log.debug("GET_CHILDREN: %s %s:%s", agent_id, desc_domain, selector) if not self._check_agent_id(agent_id): return [] if not format_check.is_valid_domain(desc_domain): return [] if not format_check.is_valid_fullselector(selector): return [] return list( self.store.get_children(str(desc_domain), str(selector), serializer=serializer, recurse=bool(recurse)))
def find_by_selector(self, agent_id, desc_domain, selector_prefix, limit=0, offset=0): log.debug("FINDBYVALUE: %s %s %s (max %d skip %d)", agent_id, desc_domain, selector_prefix, limit, offset) if not format_check.is_valid_domain(desc_domain): return [] descs = self.store.find_by_selector(str(desc_domain), str(selector_prefix), int(limit), int(offset)) return [desc.serialize_meta(serializer) for desc in descs]
def lock(self, agent_id, lockid, desc_domain, selector): if not format_check.is_valid_domain(desc_domain): return False if not format_check.is_valid_fullselector(selector): return False objpath = self.clients[agent_id] locks = self.locks[desc_domain] key = (lockid, selector) log.debug("LOCK:%s %s(%s) => %r %s:%s ", lockid, objpath, agent_id, key in locks, desc_domain, selector) if key in locks: return False locks.add(key) return True
def _pathFromSelector(self, domain, selector): """ Returns full path, from domain and selector Checks selector & domain sanity (character whitelist) Returns None if parameters were invalid. """ if not format_check.is_valid_fullselector(selector): log.error( "Provided selector (hex: %s) contains forbidden " "characters", selector.encode('hex')) return if not format_check.is_valid_domain(domain): log.error( "Provided domain (hex: %s) contains forbidden " "characters", domain.encode('hex')) return if '/%' not in selector: # no trailing '/' before the hash - add it selector = selector.replace('%', '/%') path = os.path.join(self.basepath, domain, selector[1:]) return path
def __init__(self, label, selector, value=None, domain="default", agent=None, precursors=None, version=0, processing_time=-1, uuid=None, bus=None): self.label = label """ :param label: descriptor's label. Usually a file name, or human-understandable handle :param selector: descriptor's selector :param value: descriptor's value :param domain: descriptor's domain :param agent: descriptor's agent :param precursors: descriptor's precursors :param version: descriptor's version :param processing_time: descriptor's processing_time (automatically set in most cases) :param uuid: descriptor's uuid :param bus: descriptor's bus. None iif the value must be fetched from the bus May raise a ValueError if provided selector or descriptor are invalid """ #: contains a list of parent descriptors' selectors. Typically contains #: 0 (ex. injected binaries), or (version+1) values self.precursors = precursors if precursors is not None else [] self.agent = agent self.bus = bus if not format_check.is_valid_domain(domain): raise ValueError("invalid domain format (%s)" % domain) if not format_check.is_valid_selector(selector): raise ValueError("invalid selector format (%s)" % selector) p = selector.find("%") if p >= 0: h = selector[(p+1):] self.hash = h else: if self.agent and self.precursors: if type(value) is unicode: strvalue = value.encode('utf-8') else: strvalue = str(value) v = str(self.agent) + str(self.precursors) + selector + \ strvalue else: v = str(value) self.hash = hashlib.sha256(v).hexdigest() selector = os.path.join(selector, "%" + self.hash) self.selector = selector if self.bus is None: self.value = value self.domain = domain self.version = version #: if -1, will be set by agent when push() is called self.processing_time = processing_time if uuid is None: uuid = str(m_uuid.uuid5(self.NAMESPACE_REBUS, self.hash)) #: A new uuid is generated for: #: #: * newly injected descriptors #: * descriptors that will have several versions #: * new versions of descriptors self.uuid = uuid