def process(self, descriptor, sender_id):
import tarfile
data = descriptor.value
selector = descriptor.selector
#: List of (unarchived file name, descriptor label, unarchived file
#: contents)
unarchived = []
def do_untar(archive, mode, archive_label=descriptor.label,
unarchived=unarchived):
tar = tarfile.open(fileobj=StringIO(archive), mode=mode)
for finfo in tar.getmembers():
if finfo.isfile() and finfo.size > 0:
fname = os.path.basename(finfo.name)
unarchived.append((fname, archive_label + ":" + fname,
tar.extractfile(finfo).read()))
# Compressed files
if "/compressed/bzip2" in selector:
# Try and extract - might be a .tar.bz2
try:
do_untar(descriptor.value, "r:bz2")
except tarfile.TarError:
# Probably not a compressed tar file
import bz2
data = bz2.decompress(descriptor.value)
fname = descriptor.label
if fname.endswith('.bz2'):
fname = fname[:-4]
else:
fname = "bunzipped %s" % fname
unarchived.append((fname, fname, data))
if "/compressed/gzip" in selector:
# Try and extract - might be a .tar.gz
try:
do_untar(descriptor.value, "r:gz")
except tarfile.TarError:
# Probably not a compressed tar file
from gzip import GzipFile
data = GzipFile(fileobj=StringIO(descriptor.value),
mode='rb').read()
fname = descriptor.label
if fname.endswith('.gz'):
fname = fname[:-3]
else:
fname = "gunzipped %s" % fname
unarchived.append((fname, fname, data))
# Archive files
if "/archive/tar" in selector:
do_untar(descriptor.value, mode=None)
if "/archive/zip" in selector:
from zipfile import ZipFile
fzip = ZipFile(file=StringIO(descriptor.value))
try:
for zfileinfo in fzip.filelist:
fname = zfileinfo.filename
zfile = None
for pwd in self.passwords:
try:
zfile = fzip.open(fname, pwd=pwd)
break
except RuntimeError:
# incorrect password
continue
if zfile:
unarchived.append(
(fname, descriptor.label + ':' + fname,
zfile.read()))
else:
self.log.warning(
"Could not extract %s from %s "
"(incorrect password)", fname, descriptor.label)
except RuntimeError as e:
self.log.error(e)
if "/archive/cab" in selector and self.cabextract:
try:
tmpdir = mkdtemp("rebus-cabextract")
with NamedTemporaryFile(prefix="rebus-cab") as cabfile:
cabfile.write(descriptor.value)
cabfile.flush()
try:
subprocess.check_output([self.cabextract, '-d', tmpdir,
cabfile.name],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
self.log.error("cabextract exited with status %d" %
e.returncode)
self.log.error(e.output)
for fname in os.listdir(tmpdir):
filepathname = os.path.join(tmpdir, fname)
unarchived.append((fname, descriptor.label + ':' + fname,
open(filepathname, 'rb').read()))
finally:
shutil.rmtree(tmpdir)
for fname, desclabel, fcontents in unarchived:
selector = guess_selector(buf=fcontents, label=desclabel)
desc = Descriptor(desclabel, selector, fcontents,
descriptor.domain, agent=self._name_)
self.push(desc)
self.declare_link(
descriptor, desc, "unarchived", "\"%s\" has been unarchived "
"from \"%s\"" % (fname, descriptor.label))
def run(self):
start = time.time()
def ensure_link(x):
if x.startswith("/link/"):
return x
if x.startswith("/"):
return "/link" + x
return "/link/" + x
sels = chain(*[
map(str,
self.find(self.domain, ensure_link(s), self.config['limit']))
for s in self.config['selectors']
])
class Component(object):
def __init__(self, linktype):
self.linktype = linktype
self.nodes = set()
def add(self, v):
self.nodes.add(v)
links = {}
labels = {}
def nodenamer(fmt="node%i"):
i = 0
while True:
yield fmt % i
i += 1
for s in sels:
link = self.get(self.domain, s)
uu1, uu2 = link.uuid, link.value["otherUUID"]
linktype = link.value["linktype"]
labels[uu1] = link.label
labels[uu2] = link.value["otherlabel"]
component = links.get((uu1, linktype)) or links.get(
(uu2, linktype))
if not component:
component = Component(linktype)
component.add(uu1)
component.add(uu2)
links[uu1, linktype] = links[uu2, linktype] = component
ltname = nodenamer()
dot = ['graph "links" {']
for n, l in labels.iteritems():
dot.append(
'\t"%s" [ label="%s", fontsize=10, fillcolor="#dddddd", style=filled, shape=note, href="/analysis/%s/%s"];'
% (n, l, self.domain, n))
dot.append("")
for comp in set(links.values()):
compname = ltname.next()
dot.append(
'\t"%s" [ label="%s", fontsize=8, fillcolor="#%s", style=filled, shape=oval];'
% (compname, comp.linktype,
_color_scheme.get_as_hex(comp.linktype)))
for elt in comp.nodes:
dot.append('\t"%s" -- "%s" [ len=2 ];' % (compname, elt))
dot.append("")
dot.append("}")
done = time.time()
desc = Descriptor(label="linkgraph",
selector="/graph/dot/linkgraph",
value="\n".join(dot),
domain=self.domain,
agent=self._name_,
processing_time=done - start)
self.push(desc)