def find_by_value(self, agent_id, desc_domain, selector_prefix,
value_regex):
dlist = self.iface.find_by_value(str(agent_id), desc_domain,
selector_prefix, value_regex)
return [
Descriptor.unserialize(serializer, str(s), bus=self) for s in dlist
]
def push(self, agent_id, serialized_descriptor):
descriptor = Descriptor.unserialize(serializer,
str(serialized_descriptor))
desc_domain = str(descriptor.domain)
uuid = str(descriptor.uuid)
selector = str(descriptor.selector)
# ensure processing terminates
if not format_check.processing_depth(self.store, descriptor):
log.warning(
"Refusing descriptor %s:%s received from %s: loop or "
">2 ancestors having the same descriptor", agent_id,
desc_domain, selector)
return False
if self.store.add(descriptor):
self.descriptor_count += 1
log.debug("PUSH: %s => %s:%s", agent_id, desc_domain, selector)
if not self.exiting:
self.new_descriptor(agent_id, desc_domain, uuid, selector)
# useful in case all agents are in idle/interactive mode
self._check_idle()
return True
else:
log.debug("PUSH: %s already seen => %s:%s", agent_id, desc_domain,
selector)
return False
def find_by_selector(self,
agent_id,
desc_domain,
selector_prefix,
limit=0,
offset=0):
dlist = self.iface.find_by_selector(str(agent_id), desc_domain,
selector_prefix, limit, offset)
return [
Descriptor.unserialize(serializer, str(s), bus=self) for s in dlist
]
def get_descriptor(self, domain, selector):
"""
Returns descriptor metadata, None if descriptor was not found.
"""
selector = self._version_lookup(domain, selector)
if not selector:
return None
fullpath = self.pathFromSelector(domain, selector) + ".meta"
if not os.path.isfile(fullpath):
return None
return Descriptor.unserialize(store_serializer,
open(fullpath, "rb").read())
def push(self, agent_id, serialized_descriptor):
descriptor = Descriptor.unserialize(serializer,
str(serialized_descriptor))
desc_domain = str(descriptor.domain)
uuid = str(descriptor.uuid)
selector = str(descriptor.selector)
if self.store.add(descriptor):
self.descriptor_count += 1
log.debug("PUSH: %s => %s:%s", agent_id, desc_domain, selector)
if not self.exiting:
self.new_descriptor(agent_id, desc_domain, uuid, selector)
# useful in case all agents are in idle/interactive mode
self.check_idle()
return True
else:
log.debug("PUSH: %s already seen => %s:%s", agent_id, desc_domain,
selector)
return False
def discover(self, relpath):
"""
Recursively add existing files to storage.
self.processedlock must be acquired prior to calling this function
:param relpath: starts and ends with a '/', relative to self.basepath
"""
if relpath == '/agent_intstate/':
# Ignore internal state of agents
return
path = self.basepath + relpath
self.existing_paths.add(path)
for elem in os.listdir(path):
name = path + elem
relname = relpath + elem
if os.path.isdir(name):
self.discover(relname + '/')
elif os.path.isfile(name):
basename = name.rsplit('.', 1)[0]
if name.endswith('.value'):
# Serialized descriptor value
if not os.path.isfile(basename + '.meta'):
raise Exception(
'Missing associated metadata for %s' % relname)
elif name.endswith('.meta'):
# Serialized descriptor metadata
if not os.path.isfile(basename + '.value'):
raise Exception(
'Missing associated value for %s' % relname)
with open(name, 'rb') as fp:
try:
desc = Descriptor.unserialize(store_serializer,
fp.read())
except:
log.error(
"Could not unserialize metadata from file %s",
name)
raise
fname_selector = relname.rsplit('.')[0]
# check consistency between file name and serialized
# metadata
fname_domain = fname_selector.split('/')[1]
if fname_domain != desc.domain:
raise Exception(
'Filename domain %s does not match metadata '
'domain %s for descriptor %s' %
(fname_domain, desc.domain, fname_selector))
fname_hash = fname_selector.rsplit('%', 1)[1]
if fname_hash != desc.hash:
raise Exception(
'Filename hash %s does not match metadata hash'
' %s for descriptor %s' %
(fname_hash, desc.domain, fname_selector))
self.register_meta(desc)
elif name.endswith('.cfg') and relpath == '/':
# Bus configuration
# TODO periodically save this file. Use two file, overwrite
# oldest.
if elem == '_processed.cfg':
with open(name, 'rb') as fp:
# copy processed info to self.processed
p = store_serializer.load(fp)
for dom in p.keys():
for sel, val in p[dom].items():
self.processed[dom][sel] = val
else:
raise Exception(
'Invalid file name - %s has an invalid extension '
'(must be .value, .meta or .cfg)' % relname)
else:
raise Exception(
'Invalid file type - %s is neither a regular file nor a '
'directory' % name)
def get_children(self, agent_id, desc_domain, selector, recurse=True):
return [
Descriptor.unserialize(serializer, str(s), bus=self)
for s in self.iface.get_children(str(agent_id), desc_domain,
selector, recurse)
]
def find_by_uuid(self, agent_id, desc_domain, uuid):
dlist = self.iface.find_by_uuid(str(agent_id), desc_domain, uuid)
return [
Descriptor.unserialize(serializer, str(s), bus=self) for s in dlist
]
def get(self, agent_id, desc_domain, selector):
result = str(self.iface.get(str(agent_id), desc_domain, selector))
if result == "":
return None
return Descriptor.unserialize(serializer, str(result), bus=self)
def _discover(self, relpath):
"""
Recursively add existing files to storage.
:param relpath: starts and ends with a '/', relative to self.basepath
"""
if relpath == '/agent_intstate/':
# Ignore internal state of agents
return
path = self.basepath + relpath
self.existing_paths.add(path)
for elem in os.listdir(path):
name = path + elem
relname = relpath + elem
if os.path.isdir(name):
self._discover(relname + '/')
elif os.path.isfile(name):
basename = name.rsplit('.', 1)[0]
if name.endswith('.value'):
# Serialized descriptor value
if not os.path.isfile(basename + '.meta'):
raise Exception('Missing associated metadata for %s' %
relname)
elif name.endswith('.meta'):
# Serialized descriptor metadata
if not os.path.isfile(basename + '.value'):
raise Exception('Missing associated value for %s' %
relname)
with open(name, 'rb') as fp:
try:
desc = Descriptor.unserialize(
store_serializer, fp.read())
except:
log.error(
"Could not unserialize metadata from file %s",
name)
raise
fname_selector = relname.rsplit('.')[0]
# check consistency between file name and serialized
# metadata
fname_domain = fname_selector.split('/')[1]
if fname_domain != desc.domain:
raise Exception(
'Filename domain %s does not match metadata '
'domain %s for descriptor %s' %
(fname_domain, desc.domain, fname_selector))
fname_hash = fname_selector.rsplit('%', 1)[1]
if fname_hash != desc.hash:
raise Exception(
'Filename hash %s does not match metadata hash'
' %s for descriptor %s' %
(fname_hash, desc.domain, fname_selector))
self._register_meta(desc)
elif relpath == '/' and elem == 'diskstorage.sqlite3':
continue
elif relpath == '/' and elem == '_processed.cfg':
# Former _processed.cfg storage file
log.info(
"Importing data to sqlite3 database from "
"_processed.cfg. You should delete this file after it "
"has been imported.")
with open(name, 'rb') as fp:
p = store_serializer.load(fp)
for dom in p.keys():
for sel, valset in p[dom].items():
for agent_name, config_txt in valset:
self.db.add_processed(
dom, sel, agent_name, config_txt)
else:
raise Exception(
'Invalid file name - %s has an invalid extension '
'(must be .value, .meta or .cfg)' % relname)
else:
raise Exception(
'Invalid file type - %s is neither a regular file nor a '
'directory' % name)