def process(self, descriptor, sender_id): import tarfile data = descriptor.value selector = descriptor.selector #: List of (unarchived file name, descriptor label, unarchived file #: contents) unarchived = [] def do_untar(archive, mode, archive_label=descriptor.label, unarchived=unarchived): tar = tarfile.open(fileobj=StringIO(archive), mode=mode) for finfo in tar.getmembers(): if finfo.isfile() and finfo.size > 0: fname = os.path.basename(finfo.name) unarchived.append((fname, archive_label + ":" + fname, tar.extractfile(finfo).read())) # Compressed files if "/compressed/bzip2" in selector: # Try and extract - might be a .tar.bz2 try: do_untar(descriptor.value, "r:bz2") except tarfile.TarError: # Probably not a compressed tar file import bz2 data = bz2.decompress(descriptor.value) fname = descriptor.label if fname.endswith('.bz2'): fname = fname[:-4] else: fname = "bunzipped %s" % fname unarchived.append((fname, fname, data)) if "/compressed/gzip" in selector: # Try and extract - might be a .tar.gz try: do_untar(descriptor.value, "r:gz") except tarfile.TarError: # Probably not a compressed tar file from gzip import GzipFile data = GzipFile(fileobj=StringIO(descriptor.value), mode='rb').read() fname = descriptor.label if fname.endswith('.gz'): fname = fname[:-3] else: fname = "gunzipped %s" % fname unarchived.append((fname, fname, data)) # Archive files if "/archive/tar" in selector: do_untar(descriptor.value, mode=None) if "/archive/zip" in selector: from zipfile import ZipFile fzip = ZipFile(file=StringIO(descriptor.value)) try: for zfileinfo in fzip.filelist: fname = zfileinfo.filename zfile = None for pwd in self.passwords: try: zfile = fzip.open(fname, pwd=pwd) break except RuntimeError: # incorrect password continue if zfile: unarchived.append( (fname, descriptor.label + ':' + fname, zfile.read())) else: self.log.warning( "Could not extract %s from %s " "(incorrect password)", fname, descriptor.label) except RuntimeError as e: self.log.error(e) if "/archive/cab" in selector and self.cabextract: try: tmpdir = mkdtemp("rebus-cabextract") with NamedTemporaryFile(prefix="rebus-cab") as cabfile: cabfile.write(descriptor.value) cabfile.flush() try: subprocess.check_output([self.cabextract, '-d', tmpdir, cabfile.name], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: self.log.error("cabextract exited with status %d" % e.returncode) self.log.error(e.output) for fname in os.listdir(tmpdir): filepathname = os.path.join(tmpdir, fname) unarchived.append((fname, descriptor.label + ':' + fname, open(filepathname, 'rb').read())) finally: shutil.rmtree(tmpdir) for fname, desclabel, fcontents in unarchived: selector = guess_selector(buf=fcontents, label=desclabel) desc = Descriptor(desclabel, selector, fcontents, descriptor.domain, agent=self._name_) self.push(desc) self.declare_link( descriptor, desc, "unarchived", "\"%s\" has been unarchived " "from \"%s\"" % (fname, descriptor.label))
def run(self): start = time.time() def ensure_link(x): if x.startswith("/link/"): return x if x.startswith("/"): return "/link" + x return "/link/" + x sels = chain(*[ map(str, self.find(self.domain, ensure_link(s), self.config['limit'])) for s in self.config['selectors'] ]) class Component(object): def __init__(self, linktype): self.linktype = linktype self.nodes = set() def add(self, v): self.nodes.add(v) links = {} labels = {} def nodenamer(fmt="node%i"): i = 0 while True: yield fmt % i i += 1 for s in sels: link = self.get(self.domain, s) uu1, uu2 = link.uuid, link.value["otherUUID"] linktype = link.value["linktype"] labels[uu1] = link.label labels[uu2] = link.value["otherlabel"] component = links.get((uu1, linktype)) or links.get( (uu2, linktype)) if not component: component = Component(linktype) component.add(uu1) component.add(uu2) links[uu1, linktype] = links[uu2, linktype] = component ltname = nodenamer() dot = ['graph "links" {'] for n, l in labels.iteritems(): dot.append( '\t"%s" [ label="%s", fontsize=10, fillcolor="#dddddd", style=filled, shape=note, href="/analysis/%s/%s"];' % (n, l, self.domain, n)) dot.append("") for comp in set(links.values()): compname = ltname.next() dot.append( '\t"%s" [ label="%s", fontsize=8, fillcolor="#%s", style=filled, shape=oval];' % (compname, comp.linktype, _color_scheme.get_as_hex(comp.linktype))) for elt in comp.nodes: dot.append('\t"%s" -- "%s" [ len=2 ];' % (compname, elt)) dot.append("") dot.append("}") done = time.time() desc = Descriptor(label="linkgraph", selector="/graph/dot/linkgraph", value="\n".join(dot), domain=self.domain, agent=self._name_, processing_time=done - start) self.push(desc)