def test_process_task_range_range(self, p): mkdir(cwd(analysis=3)) for x in xrange(10, 101): mkdir(cwd(analysis=x)) process_task_range("3,5,10-100") assert p.call_count == 92 # 101-10+1 p.assert_any_call({ "id": 3, "category": "file", "target": "", "options": {}, "package": None, "custom": None, }) # We did not create an analysis directory for analysis=5. with pytest.raises(AssertionError): p.assert_any_call({ "id": 5, "category": "file", "target": "", "options": {}, "package": None, "custom": None, }) for x in xrange(10, 101): p.assert_any_call({ "id": x, "category": "file", "target": "", "options": {}, "package": None, "custom": None, })
def test_path(self, p): set_cwd(tempfile.mkdtemp(prefix=".")) cuckoo_create() mkdir(cwd(analysis=1)) def create(): mkdir(cwd("suricata", "files", analysis=1)) f = open(cwd("suricata", "files", "file.1", analysis=1), "wb") f.write("a") f = open(cwd("suricata", "eve.json", analysis=1), "wb") f.write("") f = open(cwd("suricata", "files-json.log", analysis=1), "wb") f.write(json.dumps({ "id": 1, "size": 1, "filename": "a.txt", })) open(cwd("dump.pcap", analysis=1), "wb").write("pcap") s = Suricata() s.set_path(cwd(analysis=1)) s.set_options({}) s.process_pcap_binary = create s.run()
def test_empty_reprocess(self): db.connect() mkdir(cwd(analysis=1)) process_task_range("1") assert os.path.exists(cwd("reports", "report.json", analysis=1)) obj = json.load(open(cwd("reports", "report.json", analysis=1), "rb")) assert "contact back" in obj["debug"]["errors"][0]
def test_basics(): set_cwd(tempfile.mkdtemp()) cuckoo_create() mkdir(cwd(analysis=1)) init_yara() em = ExtractManager(1) em.write_extracted("foo", "bar") filepath = cwd("extracted", "0.foo", analysis=1) assert open(filepath, "rb").read() == "bar" scr = Scripting() cmd = scr.parse_command( "powershell -e %s" % "foobar".encode("utf-16le").encode("base64") ) em.push_script({ "pid": 1, "first_seen": 2, }, cmd) filepath = cwd("extracted", "0.ps1", analysis=1) assert open(filepath, "rb").read() == "foobar" em.push_command_line( "powershell -e %s" % "world!".encode("utf-16le").encode("base64") ) filepath = cwd("extracted", "1.ps1", analysis=1) assert open(filepath, "rb").read() == "world!"
def test_bson_limit(self): set_cwd(tempfile.mkdtemp()) cuckoo_create() ba = BehaviorAnalysis() ba.set_path(cwd(analysis=1)) ba.set_task({ "id": 1, }) mkdir(cwd(analysis=1)) mkdir(cwd("logs", analysis=1)) # 256mb should be fine, right? with open(cwd("logs", "1.txt", analysis=1), "wb") as f: f.write("A"*256*1024*1024) with open(cwd("logs", "2.txt", analysis=1), "wb") as f: f.write("A"*1024*1024) assert ba.run() == {} assert sorted(list(ba._enum_logs())) == [ cwd("logs", "2.txt", analysis=1), ]
def fetch_community(branch="master", force=False, filepath=None): if filepath: buf = open(filepath, "rb").read() else: log.info("Downloading.. %s", URL % branch) r = requests.get(URL % branch) if r.status_code != 200: raise CuckooOperationalError( "Error fetching the Cuckoo Community binaries " "(status_code: %d)!" % r.status_code ) buf = r.content t = tarfile.TarFile.open(fileobj=io.BytesIO(buf), mode="r:gz") folders = { "modules/signatures": "signatures", "data/monitor": "monitor", "data/yara": "yara", "agent": "agent", "analyzer": "analyzer", } members = t.getmembers() directory = members[0].name.split("/")[0] for tarfolder, outfolder in folders.items(): mkdir(cwd(outfolder)) # E.g., "community-master/modules/signatures". name_start = "%s/%s" % (directory, tarfolder) for member in members: if not member.name.startswith(name_start) or \ name_start == member.name: continue filepath = cwd(outfolder, member.name[len(name_start)+1:]) if member.isdir(): mkdir(filepath) continue # TODO Ask for confirmation as we used to do. if os.path.exists(filepath) and not force: log.debug( "Not overwriting file which already exists: %s", member.name[len(name_start)+1:] ) continue if member.issym(): t.makelink(member, filepath) continue if not os.path.exists(os.path.dirname(filepath)): os.makedirs(os.path.dirname(filepath)) log.debug("Extracted %s..", member.name[len(name_start)+1:]) open(filepath, "wb").write(t.extractfile(member).read())
def test_main(self, p, q): p.side_effect = SystemExit(0) # Ensure that the "latest" binary value makes sense so that the # "run community command" exception is not thrown. mkdir(cwd("monitor", open(cwd("monitor", "latest")).read().strip())) main.main(("--cwd", cwd(), "-d", "--nolog"), standalone_mode=False) q.assert_called_once()
def test_mkdir(): dirpath = tempfile.mkdtemp() assert os.path.isdir(dirpath) mkdir(dirpath) assert os.path.isdir(dirpath) dirpath = tempfile.mktemp() assert not os.path.exists(dirpath) mkdir(dirpath) assert os.path.isdir(dirpath)
def test_process_task(self, q, p1, p2): mkdir(cwd(analysis=1)) p1.return_value.view_task.return_value = {} main.main( ("--cwd", cwd(), "process", "-r", "1"), standalone_mode=False ) q.assert_called_once() p2.return_value.connect.assert_called_once() p1.return_value.connect.assert_not_called()
def test_process_task_range_single(self, p): mkdir(cwd(analysis=1234)) process_task_range("1234") p.assert_called_once_with({ "id": 1234, "category": "file", "target": "", "options": {}, "package": None, "custom": None, })
def fileupload(self, handler): set_cwd(tempfile.mkdtemp()) cuckoo_create() mkdir(cwd(analysis=1)) mkdir(cwd("logs", analysis=1)) handler.storagepath = cwd(analysis=1) fu = FileUpload(handler, None) fu.init() for x in fu: pass fu.close()
def test_extract_scripts(self): set_cwd(tempfile.mkdtemp()) cuckoo_create() init_yara() mkdir(cwd(analysis=1)) ba = BehaviorAnalysis() ba.set_path(cwd(analysis=1)) ba.set_task({ "id": 1, }) es = ExtractScripts(ba) es.handle_event({ "command_line": "cmd.exe /c ping 1.2.3.4", "first_seen": 1, "pid": 1234, }) es.handle_event({ "command_line": ( "powershell.exe -e " "ZQBjAGgAbwAgACIAUgBlAGMAdQByAHMAaQB2AGUAIgA=" ), "first_seen": 2, "pid": 1235, }) assert es.run() is None e = Extracted() e.set_task(Dictionary({ "id": 1, })) out = e.run() assert out == [{ "category": "script", "first_seen": 1, "pid": 1234, "program": "cmd", "raw": cwd("extracted", "0.bat", analysis=1), "yara": [], "info": {}, }, { "category": "script", "first_seen": 2, "pid": 1235, "program": "powershell", "raw": cwd("extracted", "1.ps1", analysis=1), "yara": [], "info": {}, }] assert open(out[0]["raw"], "rb").read() == "ping 1.2.3.4" assert open(out[1]["raw"], "rb").read() == 'echo "Recursive"'
def test_all_config_written(): set_cwd(tempfile.mkdtemp()) cuckoo_create() cfg = Config.from_confdir(cwd("conf")) # This one is extra, ignore. cfg["virtualbox"].pop("honeyd", None) set_cwd(tempfile.mkdtemp()) mkdir(cwd("conf")) lookups = [] class LookupDict(dict): parents = [] def __getitem__(self, key): lookups.append(":".join(self.parents + [key])) return dict.__getitem__(key) class Template(jinja2.Template): def render(self, kw): orig_config = kw["config"] def lookup_config(s): # For some reason this is called multiple times (?). if s not in lookups: lookups.append(s) return orig_config(s) kw["config"] = lookup_config for key, value in kw.items(): if key == "config": continue for key2, value2 in value.items(): value[key2] = LookupDict(value2) value[key2].parents = [key, key2] return jinja2.Template.render(self, kw) with mock.patch("cuckoo.core.init.jinja2") as p: p.Template = Template write_cuckoo_conf(cfg) for key, value in cfg.items(): for key2, value2 in value.items(): for key3, value3 in value2.items(): assert "%s:%s:%s" % (key, key2, key3) in lookups assert sorted(lookups) == sorted(set(lookups))
def test_ident_shellcode(p): set_cwd(tempfile.mkdtemp()) cuckoo_create() mkdir(cwd("yara", "scripts")) open(cwd("yara", "scripts", "1.yar"), "wb").write(""" rule Shellcode1 { strings: $Shellcode = /=\s*((0x)?[0-9A-F]{2}\s*[,;]\s*)+/ nocase condition: all of them } """) # No Yara has been installed. if not init_yara(True): return class Shellcode1(Extractor): yara_rules = "Shellcode1" def handle_yara(self, filepath, match): sc = match.string("Shellcode", 0) self.push_shellcode( "".join(chr(int(x, 16)) for x in sc[2:-1].split(",")) ) p.return_value = Shellcode1, sc = shikata(open("tests/files/shellcode/shikata/1.bin", "rb").read()) sc = ",".join("0x%02x" % ord(ch) for ch in sc) scr = Scripting() ps1 = ("[Byte[]]$s = %s;" % sc).encode("utf-16le") cmd = scr.parse_command( "powershell -e %s" % ps1.encode("base64").replace("\n", "") ) mkdir(cwd(analysis=1)) em = ExtractManager(1) em.push_script({ "pid": 1, "first_seen": 2, }, cmd) assert len(em.items) == 2 filepath = cwd("extracted", "0.ps1", analysis=1) assert open(filepath, "rb").read().startswith("[Byte[]]$s = 0xfc") buf = open(cwd("extracted", "1.bin.txt", analysis=1), "rb").read() assert "call 0x88" in buf assert "0x00c1: push 0xc69f8957" in buf assert ".db 'www.service.chrome-up.date',0" in buf
def create(): mkdir(cwd("suricata", "files", analysis=1)) f = open(cwd("suricata", "files", "file.1", analysis=1), "wb") f.write("a") f = open(cwd("suricata", "eve.json", analysis=1), "wb") f.write("") f = open(cwd("suricata", "files-json.log", analysis=1), "wb") f.write(json.dumps({ "id": 1, "size": 1, "filename": "a.txt", }))
def test_dump_memory_unicode(self): p1 = mock.MagicMock() p1.communicate.return_value = "5.0.28r111378", "" p1.returncode = 0 p2 = mock.MagicMock() p2.wait.return_value = None mkdir(cwd(analysis=1)) task_log_start(1) init_logging(logging.DEBUG) with mock.patch("cuckoo.machinery.virtualbox.Popen") as p: p.side_effect = p1, p2 self.m.dump_memory("label", u"mem\u202eory.dmp") task_log_stop(1)
def test_cfgextr(): set_cwd(tempfile.mkdtemp()) cuckoo_create() class Trigger1(Extractor): yara_rules = "Trigger1" def handle_yara(self, filepath, match): self.push_config({ "family": "barfoo", "version": "baz", }) ExtractManager.init_once() mkdir(cwd(analysis=1)) em = ExtractManager(1) em.handle_yara(None, YaraMatch({ "name": "Trigger1", "meta": None, "offsets": None, "strings": [], })) assert len(em.items) == 1 results = { "extracted": em.results(), "metadata": {}, "info": {}, } RunSignatures(results).run() assert results == { "info": { "score": 10.0, }, "metadata": { "cfgextr": [{ "family": "barfoo", "version": "baz", }], }, "extracted": mock.ANY, "signatures": [], }
def test_env(): path = tempfile.mkstemp()[1] os.environ["CUCKOO_FOOBAR"] = "top" os.environ["FOOBAR"] = "kek" mkdir(cwd("footopbar")) open(path, "wb").write(ENV_EXAMPLE) c = Config("cuckoo", cfg=path) assert c.get("cuckoo")["tmppath"] == cwd() + "/footopbar" open(path, "wb").write(ENV2_EXAMPLE) with pytest.raises(CuckooConfigurationError) as e: Config("cuckoo", cfg=path) e.match("Missing environment variable") os.environ.pop("FOOBAR") os.environ.pop("CUCKOO_FOOBAR")
def test_yes_sorted_pcap(self): set_cwd(tempfile.mkdtemp()) cuckoo_create({ "cuckoo": { "network": { "sort_pcap": True, }, }, }) mkdir(cwd(analysis=1)) shutil.copy( "tests/files/sample_analysis_storage/dump.pcap", cwd("dump.pcap", analysis=1) ) na = NetworkAnalysis() na.set_options({}) na.set_path(cwd(analysis=1)) na.run() assert os.path.exists(cwd("dump_sorted.pcap", analysis=1))
def test_static_extracted(): set_cwd(tempfile.mkdtemp()) cuckoo_create(cfg={ "processing": { "analysisinfo": { "enabled": False, }, "debug": { "enabled": False, } }, }) mkdir(cwd(analysis=1)) shutil.copy("tests/files/createproc1.docm", cwd("binary", analysis=1)) open(cwd("yara", "office", "ole.yar"), "wb").write(""" rule OleInside { strings: $s1 = "Win32_Process" condition: filename matches /word\/vbaProject.bin/ and $s1 } """) init_yara() class OleInsideExtractor(Extractor): def handle_yara(self, filepath, match): return ( match.category == "office" and match.yara[0].name == "OleInside" ) ExtractManager._instances = {} ExtractManager.extractors = OleInsideExtractor, results = RunProcessing(Dictionary({ "id": 1, "category": "file", "target": "tests/files/createproc1.docm", })).run() assert len(results["extracted"]) == 1
def test_empty_pcap(self, p): set_cwd(tempfile.mkdtemp()) cuckoo_create(cfg={ "cuckoo": { "processing": { "sort_pcap": True, }, }, }) mkdir(cwd(analysis=1)) shutil.copy( "tests/files/pcap/empty.pcap", cwd("dump.pcap", analysis=1) ) na = NetworkAnalysis() na.set_path(cwd(analysis=1)) na.set_options({}) na.run() p.warning.assert_not_called()
def test_process_task_range_multi(self, p): mkdir(cwd(analysis=1234)) mkdir(cwd(analysis=2345)) process_task_range("1234,2345") assert p.call_count == 2 p.assert_any_call({ "id": 1234, "category": "file", "target": "", "options": {}, "package": None, "custom": None, }) p.assert_any_call({ "id": 2345, "category": "file", "target": "", "options": {}, "package": None, "custom": None, })
def test_process_task_range_multi(self, p): mkdir(cwd(analysis=1234)) mkdir(cwd(analysis=2345)) process_task_range("1234,2345") assert p.call_count == 2 p.assert_any_call({ "id": 1234, "category": "file", "target": "", "options": {}, "package": None, "custom": None, }) p.assert_any_call({ "id": 2345, "category": "file", "target": "", "options": {}, "package": None, "custom": None, })
def ensure_tmpdir(): """Verifies if the current user can read and create files in the cuckoo temporary directory (and creates it, if needed).""" try: if not os.path.isdir(temppath()): mkdir(temppath()) except OSError as e: # Currently we only handle EACCES. if e.errno != errno.EACCES: raise if os.path.isdir(temppath()) and os.access(temppath(), os.R_OK | os.W_OK): return True print red( "Cuckoo cannot read or write files into the temporary directory '%s'," " please make sure the user running Cuckoo has the ability to do so. " "If the directory does not yet exist and the parent directory is " "owned by root, then please create and chown the directory with root." % temppath()) return False
def test_push_script_recursive(): set_cwd(tempfile.mkdtemp()) cuckoo_create() mkdir(cwd(analysis=1)) open(cwd("yara", "office", "ole.yar"), "wb").write(""" rule OleInside { strings: $s1 = "Win32_Process" condition: filename matches /word\/vbaProject.bin/ and $s1 } """) init_yara() s = Static() s.file_path = "tests/files/createproc1.docm" s.set_task({ "id": 1, "category": "file", "target": s.file_path, "package": "doc", }) s.run() assert ExtractManager.for_task(1).results()[0]["yara"] == [{ "name": "OleInside", "meta": { "description": "(no description)", }, "offsets": { "s1": [ (3933, 0), ], }, "strings": [ "Win32_Process".encode("base64").strip(), ], }]
def test_bson_limit(self): set_cwd(tempfile.mkdtemp()) cuckoo_create() ba = BehaviorAnalysis() ba.set_path(cwd(analysis=1)) mkdir(cwd(analysis=1)) mkdir(cwd("logs", analysis=1)) # 256mb should be fine, right? with open(cwd("logs", "1.txt", analysis=1), "wb") as f: f.write("A" * 256 * 1024 * 1024) with open(cwd("logs", "2.txt", analysis=1), "wb") as f: f.write("A" * 1024 * 1024) assert ba.run() == {} assert sorted(list(ba._enum_logs())) == [ cwd("logs", "2.txt", analysis=1), ]
def test_logger(self, p, q, r): mkdir(cwd(analysis=123)) process_task({ "id": 123, "target": "foo", "category": "bar", "package": "baz", "options": { "a": "b", }, "custom": "foobar", }) p.assert_called_once() assert p.call_args[1] == { "action": "task.report", "status": "pending", "target": "foo", "category": "bar", "package": "baz", "options": "a=b", "custom": "foobar", }
def ensure_tmpdir(): """Verifies if the current user can read and create files in the cuckoo temporary directory (and creates it, if needed).""" try: if not os.path.isdir(temppath()): mkdir(temppath()) except OSError as e: # Currently we only handle EACCES. if e.errno != errno.EACCES: raise if os.path.isdir(temppath()) and os.access(temppath(), os.R_OK | os.W_OK): return True print red( "Cuckoo cannot read or write files into the temporary directory '%s'," " please make sure the user running Cuckoo has the ability to do so. " "If the directory does not yet exist and the parent directory is " "owned by root, then please create and chown the directory with root." % temppath() ) return False
def test_push_script_recursive(): set_cwd(tempfile.mkdtemp()) cuckoo_create() mkdir(cwd(analysis=1)) open(cwd("yara", "office", "ole.yar"), "wb").write(""" rule OleInside { strings: $s1 = "Win32_Process" condition: filename matches /word\/vbaProject.bin/ and $s1 } """) init_yara() s = Static() s.file_path = "tests/files/createproc1.docm" s.set_task({ "id": 1, "category": "file", "target": s.file_path, "package": "doc", }) s.run() assert ExtractManager.for_task(1).results()[0]["yara"] == [{ "name": "OleInside", "meta": { "description": "(no description)", }, "offsets": { "s1": [ (3933, 0), ], }, "strings": [ "Win32_Process".encode("base64").strip(), ], }]
def test_open_process_log_unicode(p): set_cwd(tempfile.mkdtemp()) cuckoo_create() mkdir(cwd(analysis=1)) mkdir(cwd("logs", analysis=1)) request = server = mock.MagicMock() class Handler(ResultHandler): storagepath = cwd(analysis=1) def handle(self): pass init_logging(logging.DEBUG) try: task_log_start(1) Handler(request, (None, None), server).open_process_log({ "pid": 1, "ppid": 2, "process_name": u"\u202e", "track": True, }) finally: task_log_stop(1)
def test_process_task_range_multi_db(self, p): mkdir(cwd(analysis=1234)) mkdir(cwd(analysis=2345)) p.return_value.view_task.return_value = {} process_task_range("2345") p.return_value.view_task.assert_called_once_with(2345)
def init_legacy_analyses(): dirpath = tempfile.mkdtemp() mkdir(dirpath, "storage") mkdir(dirpath, "storage", "analyses") mkdir(dirpath, "storage", "analyses", "1") mkdir(dirpath, "storage", "analyses", "1", "logs") Files.create((dirpath, "storage", "analyses", "1", "logs"), "a.txt", "a") mkdir(dirpath, "storage", "analyses", "1", "reports") Files.create((dirpath, "storage", "analyses", "1", "reports"), "b.txt", "b") mkdir(dirpath, "storage", "analyses", "2") Files.create((dirpath, "storage", "analyses", "2"), "cuckoo.log", "log") if not is_windows(): os.symlink("thisisnotanexistingfile", os.path.join(dirpath, "storage", "analyses", "2", "binary")) Files.create((dirpath, "storage", "analyses"), "latest", "last!!1") return dirpath
def _initialize_check(self): """Runs all checks when a machine manager is initialized. @raise CuckooCriticalError: if VirtualBox Web Service is not available remotely or some configuration variables are not set or wrong. """ if not self.options.virtualbox_websrv.url: raise CuckooCriticalError( "VirtualBox Web Service URL is missing, please add it to the " "virtualbox_websrv.conf configuration file!") if not self.options.virtualbox_websrv.remote_storage: raise CuckooCriticalError( "VirtualBox host path is missing, please add remote_storage " "to the virtualbox_websrv.conf configuration file!") if not os.access(cwd("storage"), os.F_OK | os.W_OK | os.X_OK): raise CuckooCriticalError( "Not enough permissions to work with remote storage") mkdir(cwd("storage", "analyses")) mkdir(cwd("storage", "binaries")) if self.options.virtualbox_websrv.mode not in ("gui", "headless"): raise CuckooCriticalError( "VirtualBox has been configured to run in a non-supported " "mode: %s. Please upgrade your configuration to reflect " "either 'gui' or 'headless' mode!" % self.options.virtualbox_websrv.mode) if not self.options.virtualbox_websrv.debug: logging.getLogger("zeep").setLevel(logging.INFO) self.user = self.options.virtualbox_websrv.user or "" self.password = self.options.virtualbox_websrv.password or "" vbox = self._connect() super(VirtualBoxRemote, self)._initialize_check() # Restore each virtual machine to its snapshot. This will crash early # for users that don't have proper snapshots in-place, which is good. machines = vbox.list_machines() if len(machines) == 0: raise CuckooCriticalError("No virtual machines available") for machine in self.machines(): if machine.label not in machines: continue vmachine = self._get_machine(vbox, machine.label) try: if machine.snapshot: vmachine.restore(machine.snapshot) else: vmachine.restore() except remotevbox.exceptions.MachineSnapshotNX as e: raise CuckooCriticalError( "No current snapshot or a specified snapshot " "not found: %s" % e) vbox.disconnect()
from cuckoo.misc import is_windows, is_linux, is_macosx, getuser, mkdir # Note that collect_ignore is a parameter for pytest so that it knows which # unit tests to skip etc. In other words, perform platform-specific unit tests # (in terms of the Cuckoo Analyzer) depending on the current host machine. collect_ignore = [] if is_windows(): sys.path.insert(0, "cuckoo/data/analyzer/windows") collect_ignore.append("tests/linux") collect_ignore.append("tests/darwin") # Copy over the monitoring binaries as if we were in a real analysis. monitor = open("cuckoo/data/monitor/latest", "rb").read().strip() for filename in os.listdir("cuckoo/data/monitor/%s" % monitor): shutil.copy("cuckoo/data/monitor/%s/%s" % (monitor, filename), "cuckoo/data/analyzer/windows/bin/%s" % filename) if is_linux(): sys.path.insert(0, "cuckoo/data/analyzer/linux") collect_ignore.append("tests/windows") collect_ignore.append("tests/darwin") if is_macosx(): sys.path.insert(0, "cuckoo/data/analyzer/darwin") collect_ignore.append("tests/windows") collect_ignore.append("tests/linux") # Ensure the Cuckoo TMP dir exists, as some tests rely on it. mkdir(os.path.join(tempfile.gettempdir(), "cuckoo-tmp-%s" % getuser()))
def setup(self): set_cwd(tempfile.mkdtemp()) mkdir(cwd("conf")) self.h = mock.patch("cuckoo.core.init.jinja2") self.p = self.h.start() self.render().return_value = ""
def setup(self): set_cwd(tempfile.mkdtemp()) mkdir(cwd("conf")) self.h = mock.patch("cuckoo.core.init.jinja2") self.p = self.h.start() self.render().return_value = ""
def cuckoo_init(level, ctx, cfg=None): """Initialize Cuckoo configuration. @param quiet: enable quiet mode. """ logo() # It would appear this is the first time Cuckoo is being run (on this # Cuckoo Working Directory anyway). if not os.path.isdir(cwd()) or not os.listdir(cwd()): cuckoo_create(ctx.user, cfg) sys.exit(0) # Determine if this is a proper CWD. if not os.path.exists(cwd(".cwd")): sys.exit( "No proper Cuckoo Working Directory was identified, did you pass " "along the correct directory? For new installations please use a " "non-existant directory to build up the CWD! You can craft a CWD " "manually, but keep in mind that the CWD layout may change along " "with Cuckoo releases (and don't forget to fill out '$CWD/.cwd')!" ) init_console_logging(level) # Only one Cuckoo process should exist per CWD. Run this check before any # files are possibly modified. Note that we mkdir $CWD/pidfiles/ here as # its CWD migration rules only kick in after the pidfile check. mkdir(cwd("pidfiles")) pidfile = Pidfile("cuckoo") if pidfile.exists(): log.error(red("Cuckoo is already running. PID: %s"), pidfile.pid) sys.exit(1) pidfile.create() check_configs() check_version() ctx.log and init_logging(level) # Determine if any CWD updates are required and if so, do them. current = open(cwd(".cwd"), "rb").read().strip() latest = open(cwd(".cwd", private=True), "rb").read().strip() if current != latest: migrate_cwd() open(cwd(".cwd"), "wb").write(latest) Database().connect() # Load additional Signatures. load_signatures() init_modules() init_tasks() init_yara() init_binaries() init_rooter() init_routing() signatures = 0 for sig in cuckoo.signatures: if not sig.enabled: continue signatures += 1 if not signatures: log.warning( "It appears that you haven't loaded any Cuckoo Signatures. " "Signatures are highly recommended and improve & enrich the " "information extracted during an analysis. They also make up " "for the analysis score that you see in the Web Interface - so, " "pretty important!" ) log.warning( "You'll be able to fetch all the latest Cuckoo Signaturs, Yara " "rules, and more goodies by running the following command:" ) raw = cwd(raw=True) if raw == "." or raw == "~/.cuckoo": command = "cuckoo community" elif " " in raw or "'" in raw: command = 'cuckoo --cwd "%s" community' % raw else: command = "cuckoo --cwd %s community" % raw log.info("$ %s", green(command))
def init_legacy_analyses(): dirpath = tempfile.mkdtemp() mkdir(dirpath, "storage") mkdir(dirpath, "storage", "analyses") mkdir(dirpath, "storage", "analyses", "1") mkdir(dirpath, "storage", "analyses", "1", "logs") Files.create((dirpath, "storage", "analyses", "1", "logs"), "a.txt", "a") mkdir(dirpath, "storage", "analyses", "1", "reports") Files.create((dirpath, "storage", "analyses", "1", "reports"), "b.txt", "b") mkdir(dirpath, "storage", "analyses", "2") Files.create((dirpath, "storage", "analyses", "2"), "cuckoo.log", "log") Files.create((dirpath, "storage", "analyses"), "latest", "last!!1") return dirpath
def test_on_yara(): set_cwd(os.path.realpath(tempfile.mkdtemp())) cuckoo_create() init_modules() shutil.copy(cwd("yara", "binaries", "vmdetect.yar"), cwd("yara", "memory", "vmdetect.yar")) init_yara() mkdir(cwd(analysis=1)) open(cwd("binary", analysis=1), "wb").write("\x0f\x3f\x07\x0b") mkdir(cwd("files", analysis=1)) open(cwd("files", "1.txt", analysis=1), "wb").write("\x56\x4d\x58\x68") mkdir(cwd("memory", analysis=1)) open(cwd("memory", "1-0.dmp", analysis=1), "wb").write( struct.pack("QIIII", 0x400000, 0x1000, 0, 0, 0) + "\x45\xc7\x00\x01") Database().connect() ExtractManager._instances = {} results = RunProcessing(task=Dictionary({ "id": 1, "category": "file", "target": __file__, })).run() assert results["target"]["file"]["yara"][0]["offsets"] == { "virtualpc": [(0, 0)], } assert results["procmemory"][0]["regions"] == [{ "addr": "0x00400000", "end": "0x00401000", "offset": 24, "protect": None, "size": 4096, "state": 0, "type": 0, }] assert results["procmemory"][0]["yara"][0]["offsets"] == { "vmcheckdll": [(24, 0)], } assert results["dropped"][0]["yara"][0]["offsets"] == { "vmware": [(0, 0)], "vmware1": [(0, 0)], } class sig1(Signature): name = "sig1" @property def matched(self): return False @matched.setter def matched(self, value): pass def init(self): pass def on_signature(self, sig): pass def on_complete(self): pass def on_extract(self, match): pass on_yara = mock.MagicMock() rs = RunSignatures(results) rs.signatures = sig1(rs), rs.run() assert sig1.on_yara.call_count == 3 sig1.on_yara.assert_any_call("sample", cwd("binary", analysis=1), mock.ANY) sig1.on_yara.assert_any_call("dropped", cwd("files", "1.txt", analysis=1), mock.ANY) sig1.on_yara.assert_any_call("procmem", cwd("memory", "1-0.dmp", analysis=1), mock.ANY) ym = sig1.on_yara.call_args_list[0][0][2] assert ym.offsets == { "virtualpc": [(0, 0)], } assert ym.string("virtualpc", 0) == "\x0f\x3f\x07\x0b"
def cuckoo_init(level, ctx, cfg=None): """Initialize Cuckoo configuration. @param quiet: enable quiet mode. """ logo() # It would appear this is the first time Cuckoo is being run (on this # Cuckoo Working Directory anyway). if not os.path.isdir(cwd()) or not os.listdir(cwd()): cuckoo_create(ctx.user, cfg) sys.exit(0) # Determine if this is a proper CWD. if not os.path.exists(cwd(".cwd")): sys.exit( "No proper Cuckoo Working Directory was identified, did you pass " "along the correct directory? For new installations please use a " "non-existant directory to build up the CWD! You can craft a CWD " "manually, but keep in mind that the CWD layout may change along " "with Cuckoo releases (and don't forget to fill out '$CWD/.cwd')!") init_console_logging(level) # Only one Cuckoo process should exist per CWD. Run this check before any # files are possibly modified. Note that we mkdir $CWD/pidfiles/ here as # its CWD migration rules only kick in after the pidfile check. mkdir(cwd("pidfiles")) pidfile = Pidfile("cuckoo") if pidfile.exists(): log.error(red("Cuckoo is already running. PID: %s"), pidfile.pid) sys.exit(1) pidfile.create() check_configs() check_version() ctx.log and init_logging(level) # Determine if any CWD updates are required and if so, do them. current = open(cwd(".cwd"), "rb").read().strip() latest = open(cwd(".cwd", private=True), "rb").read().strip() if current != latest: migrate_cwd() open(cwd(".cwd"), "wb").write(latest) # Ensure the user is able to create and read temporary files. if not ensure_tmpdir(): sys.exit(1) Database().connect() # Load additional Signatures. load_signatures() init_modules() init_tasks() init_yara() init_binaries() init_rooter() init_routing() signatures = 0 for sig in cuckoo.signatures: if not sig.enabled: continue signatures += 1 if not signatures: log.warning( "It appears that you haven't loaded any Cuckoo Signatures. " "Signatures are highly recommended and improve & enrich the " "information extracted during an analysis. They also make up " "for the analysis score that you see in the Web Interface - so, " "pretty important!") log.warning( "You'll be able to fetch all the latest Cuckoo Signaturs, Yara " "rules, and more goodies by running the following command:") log.info("$ %s", green(format_command("community")))
def init_analysis(task_id, package, *filename): """Initialize an analysis with an "encrypted" binary from tests/files/.""" mkdir(cwd(analysis=task_id)) content = open(os.path.join("tests", "files", *filename), "rb").read() open(cwd("binary", analysis=task_id), "wb").write(content[::-1])
def test_on_extract(): set_cwd(tempfile.mkdtemp()) cuckoo_create() init_modules() Database().connect() mkdir(cwd(analysis=2)) cmd = Scripting().parse_command("cmd.exe /c ping 1.2.3.4") ex = ExtractManager.for_task(2) ex.push_script({ "pid": 1, "first_seen": 2, }, cmd) results = RunProcessing(task=Dictionary({ "id": 2, "category": "file", "target": __file__, })).run() assert results["extracted"] == [{ "category": "script", "pid": 1, "first_seen": 2, "program": "cmd", "raw": cwd("extracted", "0.bat", analysis=2), "yara": [], "info": {}, }] class sig1(object): name = "sig1" @property def matched(self): return False @matched.setter def matched(self, value): pass def init(self): pass def on_signature(self): pass def on_complete(self): pass def on_yara(self): pass on_extract = mock.MagicMock() rs = RunSignatures(results) rs.signatures = sig1(), rs.run() sig1.on_extract.assert_called_once() em = sig1.on_extract.call_args_list[0][0][0] assert em.category == "script"
def init_legacy_analyses(): dirpath = tempfile.mkdtemp() mkdir(dirpath, "storage") mkdir(dirpath, "storage", "analyses") mkdir(dirpath, "storage", "analyses", "1") mkdir(dirpath, "storage", "analyses", "1", "logs") Files.create( (dirpath, "storage", "analyses", "1", "logs"), "a.txt", "a" ) mkdir(dirpath, "storage", "analyses", "1", "reports") Files.create( (dirpath, "storage", "analyses", "1", "reports"), "b.txt", "b" ) mkdir(dirpath, "storage", "analyses", "2") Files.create((dirpath, "storage", "analyses", "2"), "cuckoo.log", "log") Files.create((dirpath, "storage", "analyses"), "latest", "last!!1") return dirpath
def test_stap_log(self): set_cwd(tempfile.mkdtemp()) cuckoo_create() init_yara() mkdir(cwd(analysis=1)) mkdir(cwd("logs", analysis=1)) shutil.copy( "tests/files/log_full.stap", cwd("logs", "all.stap", analysis=1) ) ba = BehaviorAnalysis() ba.set_path(cwd(analysis=1)) ba.set_task({ "id": 1, }) assert ba.run() == { "generic": [{ "first_seen": datetime.datetime(2017, 8, 28, 14, 29, 32, 618541), "pid": 820, "ppid": 819, "process_name": "sh", "process_path": None, "summary": {}, }, { "first_seen": datetime.datetime(2017, 8, 28, 14, 29, 32, 619135), "pid": 821, "ppid": 820, "process_name": "bash", "process_path": None, "summary": {}, }, { "first_seen": datetime.datetime(2017, 8, 28, 14, 29, 32, 646318), "pid": 822, "ppid": 821, "process_name": "ls", "process_path": None, "summary": {}, }], "processes": [{ "calls": [], "command_line": "/bin/sh /tmp/execve.sh", "first_seen": datetime.datetime(2017, 8, 28, 14, 29, 32, 618541), "pid": 820, "ppid": 819, "process_name": "sh", "type": "process" }, { "calls": [], "command_line": ( "/bin/bash -c python -c 'import subprocess; " "subprocess.call([\"/bin/ls\", \"/hax\"])'" ), "first_seen": datetime.datetime(2017, 8, 28, 14, 29, 32, 619135), "pid": 821, "ppid": 820, "process_name": "bash", "type": "process" }, { "calls": [], "command_line": "/bin/ls /hax", "first_seen": datetime.datetime(2017, 8, 28, 14, 29, 32, 646318), "pid": 822, "ppid": 821, "process_name": "ls", "type": "process" }], "processtree": [{ "children": [{ "children": [{ "children": [], "command_line": "/bin/ls /hax", "first_seen": datetime.datetime(2017, 8, 28, 14, 29, 32, 646318), "pid": 822, "ppid": 821, "process_name": "ls", "track": True }], "command_line": ( "/bin/bash -c python -c 'import subprocess; " "subprocess.call([\"/bin/ls\", \"/hax\"])'" ), "first_seen": datetime.datetime(2017, 8, 28, 14, 29, 32, 619135), "pid": 821, "ppid": 820, "process_name": "bash", "track": True }], "command_line": "/bin/sh /tmp/execve.sh", "first_seen": datetime.datetime(2017, 8, 28, 14, 29, 32, 618541), "pid": 820, "ppid": 819, "process_name": "sh", "track": True }], }
def init_analysis(task_id, package, *filename): """Initializes an analysis with an "encrypted" binary from tests/files/.""" mkdir(cwd(analysis=task_id)) content = open(os.path.join("tests", "files", *filename), "rb").read() open(cwd("binary", analysis=task_id), "wb").write(content[::-1])
def test_process_task_range_single_db(self, p): mkdir(cwd(analysis=1234)) p.return_value.view_task.return_value = {} process_task_range("1234") p.return_value.view_task.assert_called_once_with(1234)
def run(self): results = {"http_ex": [], "https_ex": [], "smtp_ex": []} mkdir(self.network_path) r = httpreplay.reader.PcapReader(open(self.pcap_path, "rb")) r.tcp = httpreplay.s****a.TCPPacketStreamer(r, self.handlers) l = sorted(r.process(), key=lambda x: x[1]) for s, ts, protocol, sent, recv in l: srcip, srcport, dstip, dstport = s if protocol == "smtp": results["smtp_ex"].append({ "src": srcip, "dst": dstip, "sport": srcport, "dport": dstport, "protocol": protocol, "req": { "hostname": sent.hostname, "mail_from": sent.mail_from, "mail_to": sent.mail_to, "auth_type": sent.auth_type, "username": sent.username, "password": sent.password, "headers": sent.headers, "mail_body": sent.message }, "resp": { "banner": recv.ready_message } }) if protocol == "http" or protocol == "https": request = sent.raw.split("\r\n\r\n", 1)[0] response = recv.raw.split("\r\n\r\n", 1)[0] # TODO Don't create empty files (e.g., the sent body for a # GET request or a 301/302 HTTP redirect). req_md5 = hashlib.md5(sent.body or "").hexdigest() req_sha1 = hashlib.sha1(sent.body or "").hexdigest() req_path = os.path.join(self.network_path, req_sha1) open(req_path, "wb").write(sent.body or "") resp_md5 = hashlib.md5(recv.body or "").hexdigest() resp_sha1 = hashlib.sha1(recv.body or "").hexdigest() resp_path = os.path.join(self.network_path, resp_sha1) open(resp_path, "wb").write(recv.body or "") results["%s_ex" % protocol].append({ "src": srcip, "sport": srcport, "dst": dstip, "dport": dstport, "protocol": protocol, "method": sent.method, "host": sent.headers.get("host", dstip), "uri": sent.uri, "status": int(getattr(recv, "status", 0)), # We'll keep these fields here for now. "request": request.decode("latin-1"), "response": response.decode("latin-1"), # It's not perfect yet, but it'll have to do. "req": { "path": req_path, "md5": req_md5, "sha1": req_sha1, }, "resp": { "path": resp_path, "md5": resp_md5, "sha1": resp_sha1, }, # Obsolete fields. "md5": resp_md5, "sha1": resp_sha1, "path": resp_path, }) return results
def migrate_cwd(): log.warning( "This is the first time you're running Cuckoo after updating your " "local version of Cuckoo. We're going to update files in your CWD " "that require updating. Note that we'll first ensure that no custom " "patches have been applied by you before applying any modifications " "of our own.") # Remove now-obsolete index_*.yar files. for filename in os.listdir(cwd("yara")): if filename.startswith("index_") and filename.endswith(".yar"): os.remove(cwd("yara", filename)) # Create new directories if not present yet. mkdir(cwd("stuff")) mkdir(cwd("yara", "office")) # Create the new $CWD/whitelist/ directory. if not os.path.exists(cwd("whitelist")): shutil.copytree(cwd("..", "data", "whitelist", private=True), cwd("whitelist")) hashes = {} for line in open(cwd("cwd", "hashes.txt", private=True), "rb"): if not line.strip() or line.startswith("#"): continue hash_, filename = line.split() hashes[filename] = hashes.get(filename, []) + [hash_] # We remove $CWD/monitor/latest upfront if it's a symbolic link, because # our migration code doesn't properly handle symbolic links. if os.path.islink(cwd("monitor", "latest")): os.remove(cwd("monitor", "latest")) modified, outdated, deleted = [], [], [] for filename, hashes in hashes.items(): if not os.path.exists(cwd(filename)): if hashes[-1] != "0" * 40: outdated.append(filename) continue hash_ = hashlib.sha1(open(cwd(filename), "rb").read()).hexdigest() if hash_ not in hashes: modified.append(filename) elif hashes[-1] == "0" * 40: deleted.append(filename) elif hash_ != hashes[-1]: outdated.append(filename) if modified: log.error( "One or more files in the CWD have been modified outside of " "regular Cuckoo usage. Due to these changes Cuckoo isn't able to " "automatically upgrade your setup.") for filename in sorted(modified): log.warning("Modified file: %s (=> %s)", filename, cwd(filename)) log.error("Moving forward you have two options:") log.warning( "1) You make a backup of the affected files, remove their " "presence in the CWD (yes, actually 'rm -f' the file), and " "re-run Cuckoo to automatically restore the new version of the " "file. Afterwards you'll be able to re-apply any changes as you " "like.") log.warning( "2) You revert back to the version of Cuckoo you were on " "previously and accept that manual changes that have not been " "merged upstream require additional maintenance that you'll " "pick up at a later point in time.") sys.exit(1) for filename in sorted(deleted): log.debug("Deleted %s", filename) os.unlink(cwd(filename)) for filename in sorted(outdated): filepath = cwd("..", "data", filename, private=True) if not os.path.exists(filepath): log.debug( "Failed to upgrade file not shipped with this release: %s", filename) continue log.debug("Upgraded %s", filename) if not os.path.exists(os.path.dirname(cwd(filename))): os.makedirs(os.path.dirname(cwd(filename))) shutil.copy(filepath, cwd(filename)) log.info("Automated migration of your CWD was successful! Continuing " "execution of Cuckoo as expected.")
def run(self): results = { "http_ex": [], "https_ex": [], "smtp_ex": [] } mkdir(self.network_path) r = httpreplay.reader.PcapReader(open(self.pcap_path, "rb")) r.tcp = httpreplay.s****a.TCPPacketStreamer(r, self.handlers) l = sorted(r.process(), key=lambda x: x[1]) for s, ts, protocol, sent, recv in l: srcip, srcport, dstip, dstport = s if protocol == "smtp": results["smtp_ex"].append({ "src": srcip, "dst": dstip, "sport": srcport, "dport": dstport, "protocol": protocol, "req": { "hostname": sent.hostname, "mail_from": sent.mail_from, "mail_to": sent.mail_to, "auth_type": sent.auth_type, "username": sent.username, "password": sent.password, "headers": sent.headers, "mail_body": sent.message }, "resp": { "banner": recv.ready_message } }) if protocol == "http" or protocol == "https": request = sent.raw.split("\r\n\r\n", 1)[0] response = recv.raw.split("\r\n\r\n", 1)[0] # TODO Don't create empty files (e.g., the sent body for a # GET request or a 301/302 HTTP redirect). req_md5 = hashlib.md5(sent.body or "").hexdigest() req_sha1 = hashlib.sha1(sent.body or "").hexdigest() req_path = os.path.join(self.network_path, req_sha1) open(req_path, "wb").write(sent.body or "") resp_md5 = hashlib.md5(recv.body or "").hexdigest() resp_sha1 = hashlib.sha1(recv.body or "").hexdigest() resp_path = os.path.join(self.network_path, resp_sha1) open(resp_path, "wb").write(recv.body or "") results["%s_ex" % protocol].append({ "src": srcip, "sport": srcport, "dst": dstip, "dport": dstport, "protocol": protocol, "method": sent.method, "host": sent.headers.get("host", dstip), "uri": sent.uri, "status": int(getattr(recv, "status", 0)), # We'll keep these fields here for now. "request": request.decode("latin-1"), "response": response.decode("latin-1"), # It's not perfect yet, but it'll have to do. "req": { "path": req_path, "md5": req_md5, "sha1": req_sha1, }, "resp": { "path": resp_path, "md5": resp_md5, "sha1": resp_sha1, }, # Obsolete fields. "md5": resp_md5, "sha1": resp_sha1, "path": resp_path, }) return results
def import_(self, f, submit_id): """Import an analysis identified by the file(-like) object f.""" try: z = zipfile.ZipFile(f) except zipfile.BadZipfile: raise CuckooOperationalError( "Imported analysis is not a proper .zip file.") # Ensure there are no files with illegal or potentially insecure names. # TODO Keep in mind that if we start to support other archive formats # (e.g., .tar) that those may also support symbolic links. In that case # we should probably start using sflock here. for filename in z.namelist(): if filename.startswith("/") or ".." in filename or ":" in filename: raise CuckooOperationalError( "The .zip file contains a file with a potentially " "incorrect filename: %s" % filename) if "task.json" not in z.namelist(): raise CuckooOperationalError( "The task.json file is required in order to be able to import " "an analysis! This file contains metadata about the analysis.") required_fields = { "options": dict, "route": basestring, "package": basestring, "target": basestring, "category": basestring, "memory": bool, "timeout": (int, long), "priority": (int, long), "custom": basestring, "tags": (tuple, list), } try: info = json.loads(z.read("task.json")) for key, type_ in required_fields.items(): if key not in info: raise ValueError("missing %s" % key) if info[key] is not None and not isinstance(info[key], type_): raise ValueError("%s => %s" % (key, info[key])) except ValueError as e: raise CuckooOperationalError( "The provided task.json file, required for properly importing " "the analysis, is incorrect or incomplete (%s)." % e) if info["category"] == "url": task_id = submit_task.add_url(url=info["target"], package=info["package"], timeout=info["timeout"], options=info["options"], priority=info["priority"], custom=info["custom"], memory=info["memory"], tags=info["tags"], submit_id=submit_id) else: # Users may have the "delete_bin_copy" enabled and in such cases # the binary file won't be included in the .zip file. if "binary" in z.namelist(): filepath = Files.temp_named_put( z.read("binary"), os.path.basename(info["target"])) else: # Generate a temp file as a target if no target is present filepath = Files.temp_put("") # We'll be updating the target shortly. task_id = submit_task.add_path(file_path=filepath, package=info["package"], timeout=info["timeout"], options=info["options"], priority=info["priority"], custom=info["custom"], memory=info["memory"], tags=info["tags"], submit_id=submit_id) if not task_id: raise CuckooOperationalError( "There was an error creating a task for the to-be imported " "analysis in our database.. Can't proceed.") # The constructors currently don't accept this argument. db.set_route(task_id, info["route"]) mkdir(cwd(analysis=task_id)) z.extractall(cwd(analysis=task_id)) # If there's an analysis.json file, load it up to figure out additional # metdata regarding this analysis. if os.path.exists(cwd("analysis.json", analysis=task_id)): try: obj = json.load( open(cwd("analysis.json", analysis=task_id), "rb")) if not isinstance(obj, dict): raise ValueError if "errors" in obj and not isinstance(obj["errors"], list): raise ValueError if "action" in obj and not isinstance(obj["action"], list): raise ValueError except ValueError: log.warning( "An analysis.json file was provided, but wasn't a valid " "JSON object/structure that we can to enhance the " "analysis information.") else: for error in set(obj.get("errors", [])): if isinstance(error, basestring): db.add_error(error, task_id) for action in set(obj.get("action", [])): if isinstance(action, basestring): db.add_error("", task_id, action) # We set this analysis as completed so that it will be processed # automatically (assuming 'cuckoo process' is running). db.set_status(task_id, TASK_COMPLETED) return task_id
def test_on_yara(): set_cwd(tempfile.mkdtemp()) cuckoo_create() init_modules() shutil.copy( cwd("yara", "binaries", "vmdetect.yar"), cwd("yara", "memory", "vmdetect.yar") ) init_yara(True) mkdir(cwd(analysis=1)) open(cwd("binary", analysis=1), "wb").write("\x0f\x3f\x07\x0b") mkdir(cwd("files", analysis=1)) open(cwd("files", "1.txt", analysis=1), "wb").write("\x56\x4d\x58\x68") mkdir(cwd("memory", analysis=1)) open(cwd("memory", "1-0.dmp", analysis=1), "wb").write( struct.pack("QIIII", 0x400000, 0x1000, 0, 0, 0) + "\x45\xc7\x00\x01" ) Database().connect() results = RunProcessing(task=Dictionary({ "id": 1, "category": "file", "target": __file__, })).run() assert results["target"]["file"]["yara"][0]["offsets"] == { "virtualpc": [(0, 0)], } assert results["procmemory"][0]["yara"][0]["offsets"] == { "vmcheckdll": [(24, 0)], } assert results["dropped"][0]["yara"][0]["offsets"] == { "vmware": [(0, 0)], "vmware1": [(0, 0)], } class sig1(object): name = "sig1" @property def matched(self): return False @matched.setter def matched(self, value): pass def init(self): pass def on_signature(self): pass def on_complete(self): pass on_yara = mock.MagicMock() rs = RunSignatures(results) rs.signatures = sig1(), rs.run() assert sig1.on_yara.call_count == 3 sig1.on_yara.assert_any_call( "sample", cwd("binary", analysis=1), mock.ANY ) sig1.on_yara.assert_any_call( "dropped", cwd("files", "1.txt", analysis=1), mock.ANY ) sig1.on_yara.assert_any_call( "procmem", cwd("memory", "1-0.dmp", analysis=1), mock.ANY ) assert sig1.on_yara.call_args_list[0][0][2]["offsets"] == { "virtualpc": [(0, 0)], }