def get_processable(self, agent_id, desc_domain, selector):
if not format_check.is_valid_domain(desc_domain):
return []
if not format_check.is_valid_fullselector(selector):
return []
log.debug("GET_PROCESSABLE: %s:%s %s", desc_domain, selector, agent_id)
return self.store.get_processable(str(desc_domain), str(selector))
def list_uuids(self, agent_id, desc_domain):
log.debug("LISTUUIDS: %s %s", agent_id, desc_domain)
if not self._check_agent_id(agent_id):
return {}
if not format_check.is_valid_domain(desc_domain):
return {}
return self.store.list_uuids(str(desc_domain))
def processed_stats(self, agent_id, desc_domain):
log.debug("PROCESSED_STATS: %s %s", agent_id, desc_domain)
if not self._check_agent_id(agent_id):
return []
if not format_check.is_valid_domain(desc_domain):
return []
return self.store.processed_stats(str(desc_domain))
def unlock(self, agent_id, lockid, desc_domain, selector,
processing_failed, retries, wait_time):
if not format_check.is_valid_domain(desc_domain):
return
if not format_check.is_valid_fullselector(selector):
return
objpath = self.clients[agent_id]
locks = self.locks[desc_domain]
lkey = (lockid, selector)
log.debug("UNLOCK:%s %s(%s) => %r %d:%d ", lockid, objpath, agent_id,
processing_failed, retries, wait_time)
if lkey not in locks:
return
locks.remove(lkey)
# find agent_name, config_txt
for (agent_name, config_txt), ids in self.uniq_conf_clients.items():
if agent_id in ids:
break
rkey = (agent_name, config_txt, desc_domain, selector)
if rkey not in self.retry_counters:
self.retry_counters[rkey] = retries
if self.retry_counters[rkey] > 0:
self.retry_counters[rkey] -= 1
desc = self.store.get_descriptor(desc_domain, selector)
uuid = desc.uuid
self.sched.add_action(
wait_time, (agent_id, desc_domain, uuid, selector, agent_name))
def register(self, agent_id, agent_domain, pth, config_txt,
processes_descriptors):
#: indicates whether another instance of the same agent is already
#: running with the same configuration
if not format_check.is_valid_domain(agent_domain):
return
agent_name = agent_id.split('-', 1)[0]
self.agentnames[agent_id] = agent_name
output_altering_options = get_output_altering_options(str(config_txt))
name_config = (agent_name, output_altering_options)
already_running = len(self.uniq_conf_clients[name_config]) > 1
self.uniq_conf_clients[name_config].append(agent_id)
self.clients[agent_id] = pth
self.agents_output_altering_options[agent_id] = output_altering_options
self.agents_full_config_txts[agent_id] = str(config_txt)
log.info("New client %s (%s) in domain %s with config %s", pth,
agent_id, agent_domain, config_txt)
if not processes_descriptors:
self.descriptor_handled_count[name_config] = 0
elif not already_running:
# Send not-yet processed descriptors to the agent,
# unless another instance of the same agent has already been
# started, and should be processing those descriptors
unprocessed = \
self.store.list_unprocessed_by_agent(agent_name,
output_altering_options)
self.descriptor_handled_count[name_config] = \
self.descriptor_count - len(unprocessed)
for dom, uuid, sel in unprocessed:
self.targeted_descriptor("storage", dom, uuid, sel,
[agent_name], False)
def find_by_uuid(self, agent_id, desc_domain, uuid):
log.debug("FINDBYUUID: %s %s:%s", agent_id, desc_domain, uuid)
if not self._check_agent_id(agent_id):
return []
if not format_check.is_valid_domain(desc_domain):
return []
descs = self.store.find_by_uuid(str(desc_domain), str(uuid))
return [desc.serialize_meta(serializer) for desc in descs]
def find(self, agent_id, desc_domain, selector_regex, limit=0, offset=0):
log.debug("FIND: %s %s:%s (max %d skip %d)", agent_id, desc_domain,
selector_regex, limit, offset)
if not format_check.is_valid_domain(desc_domain):
return []
selectors = self.store.find(str(desc_domain), str(selector_regex),
str(limit), int(offset))
return [str(s) for s in selectors]
def get_children(self, agent_id, desc_domain, selector, recurse):
log.debug("GET_CHILDREN: %s %s:%s", agent_id, desc_domain, selector)
if not format_check.is_valid_domain(desc_domain):
return []
if not format_check.is_valid_fullselector(selector):
return []
descs = self.store.get_children(str(desc_domain),
str(selector),
recurse=bool(recurse))
return [desc.serialize_meta(serializer) for desc in descs]
def find_by_value(self, agent_id, desc_domain, selector_prefix,
value_regex):
log.debug("FINDBYVALUE: %s %s %s %s", agent_id, desc_domain,
selector_prefix, value_regex)
if not format_check.is_valid_domain(desc_domain):
return []
descs = self.store.find_by_value(str(desc_domain),
str(selector_prefix),
str(value_regex))
return [desc.serialize_meta(serializer) for desc in descs]
def get_value(self, agent_id, desc_domain, selector):
log.debug("GETVALUE: %s %s:%s", agent_id, desc_domain, selector)
if not format_check.is_valid_domain(desc_domain):
return None
if not format_check.is_valid_selector(selector):
return None
value = self.store.get_value(str(desc_domain), str(selector))
if value is None:
return ""
return serializer.dumps(value)
def get(self, agent_id, desc_domain, selector):
log.debug("GET: %s %s:%s", agent_id, desc_domain, selector)
if not format_check.is_valid_domain(desc_domain):
return None
if not format_check.is_valid_selector(selector):
return None
desc = self.store.get_descriptor(str(desc_domain), str(selector))
if desc is None:
return ""
return desc.serialize_meta(serializer)
def request_processing(self, agent_id, desc_domain, selector, targets):
log.debug("REQUEST_PROCESSING: %s %s:%s targets %s", agent_id,
desc_domain, selector, [str(t) for t in targets])
if not format_check.is_valid_domain(desc_domain):
return
if not format_check.is_valid_fullselector(selector):
return
d = self.store.get_descriptor(str(desc_domain), str(selector))
self.userrequestid += 1
self.targeted_descriptor(agent_id, desc_domain, d.uuid, selector,
targets, self.userrequestid)
def mark_processable(self, agent_id, desc_domain, selector):
if not format_check.is_valid_domain(desc_domain):
return
if not format_check.is_valid_fullselector(selector):
return
agent_name = self.agentnames[agent_id]
options = self.agents_output_altering_options[agent_id]
log.debug("MARK_PROCESSABLE: %s:%s %s %s", desc_domain, selector,
agent_id, options)
isnew = self.store.mark_processable(str(desc_domain), str(selector),
agent_name, str(options))
if isnew:
self._update_check_idle(agent_name, options)
def get_children(self, agent_id, desc_domain, selector, recurse):
log.debug("GET_CHILDREN: %s %s:%s", agent_id, desc_domain, selector)
if not self._check_agent_id(agent_id):
return []
if not format_check.is_valid_domain(desc_domain):
return []
if not format_check.is_valid_fullselector(selector):
return []
return list(
self.store.get_children(str(desc_domain),
str(selector),
serializer=serializer,
recurse=bool(recurse)))
def find_by_selector(self,
agent_id,
desc_domain,
selector_prefix,
limit=0,
offset=0):
log.debug("FINDBYVALUE: %s %s %s (max %d skip %d)", agent_id,
desc_domain, selector_prefix, limit, offset)
if not format_check.is_valid_domain(desc_domain):
return []
descs = self.store.find_by_selector(str(desc_domain),
str(selector_prefix), int(limit),
int(offset))
return [desc.serialize_meta(serializer) for desc in descs]
def lock(self, agent_id, lockid, desc_domain, selector):
if not format_check.is_valid_domain(desc_domain):
return False
if not format_check.is_valid_fullselector(selector):
return False
objpath = self.clients[agent_id]
locks = self.locks[desc_domain]
key = (lockid, selector)
log.debug("LOCK:%s %s(%s) => %r %s:%s ", lockid, objpath, agent_id, key
in locks, desc_domain, selector)
if key in locks:
return False
locks.add(key)
return True
def _pathFromSelector(self, domain, selector):
"""
Returns full path, from domain and selector
Checks selector & domain sanity (character whitelist)
Returns None if parameters were invalid.
"""
if not format_check.is_valid_fullselector(selector):
log.error(
"Provided selector (hex: %s) contains forbidden "
"characters", selector.encode('hex'))
return
if not format_check.is_valid_domain(domain):
log.error(
"Provided domain (hex: %s) contains forbidden "
"characters", domain.encode('hex'))
return
if '/%' not in selector:
# no trailing '/' before the hash - add it
selector = selector.replace('%', '/%')
path = os.path.join(self.basepath, domain, selector[1:])
return path
def __init__(self, label, selector, value=None, domain="default",
agent=None, precursors=None, version=0, processing_time=-1,
uuid=None, bus=None):
self.label = label
"""
:param label: descriptor's label. Usually a file name, or
human-understandable handle
:param selector: descriptor's selector
:param value: descriptor's value
:param domain: descriptor's domain
:param agent: descriptor's agent
:param precursors: descriptor's precursors
:param version: descriptor's version
:param processing_time: descriptor's processing_time (automatically set
in most cases)
:param uuid: descriptor's uuid
:param bus: descriptor's bus. None iif the value must be fetched from
the bus
May raise a ValueError if provided selector or descriptor are invalid
"""
#: contains a list of parent descriptors' selectors. Typically contains
#: 0 (ex. injected binaries), or (version+1) values
self.precursors = precursors if precursors is not None else []
self.agent = agent
self.bus = bus
if not format_check.is_valid_domain(domain):
raise ValueError("invalid domain format (%s)" % domain)
if not format_check.is_valid_selector(selector):
raise ValueError("invalid selector format (%s)" % selector)
p = selector.find("%")
if p >= 0:
h = selector[(p+1):]
self.hash = h
else:
if self.agent and self.precursors:
if type(value) is unicode:
strvalue = value.encode('utf-8')
else:
strvalue = str(value)
v = str(self.agent) + str(self.precursors) + selector + \
strvalue
else:
v = str(value)
self.hash = hashlib.sha256(v).hexdigest()
selector = os.path.join(selector, "%" + self.hash)
self.selector = selector
if self.bus is None:
self.value = value
self.domain = domain
self.version = version
#: if -1, will be set by agent when push() is called
self.processing_time = processing_time
if uuid is None:
uuid = str(m_uuid.uuid5(self.NAMESPACE_REBUS, self.hash))
#: A new uuid is generated for:
#:
#: * newly injected descriptors
#: * descriptors that will have several versions
#: * new versions of descriptors
self.uuid = uuid
