def test_submit_abort(self, p, capsys):
p.side_effect = KeyboardInterrupt
main.main((
"--cwd", cwd(), "submit", Files.create(cwd(), "a.txt", "hello")
), standalone_mode=False)
out, _ = capsys.readouterr()
assert "Aborting submission of" in out
def test_empty_reprocess(self):
db.connect()
mkdir(cwd(analysis=1))
process_task_range("1")
assert os.path.exists(cwd("reports", "report.json", analysis=1))
obj = json.load(open(cwd("reports", "report.json", analysis=1), "rb"))
assert "contact back" in obj["debug"]["errors"][0]
def test_process_task_range_range(self, p):
mkdir(cwd(analysis=3))
for x in xrange(10, 101):
mkdir(cwd(analysis=x))
process_task_range("3,5,10-100")
assert p.call_count == 92 # 101-10+1
p.assert_any_call({
"id": 3,
"category": "file",
"target": "",
"options": {},
"package": None,
"custom": None,
})
# We did not create an analysis directory for analysis=5.
with pytest.raises(AssertionError):
p.assert_any_call({
"id": 5,
"category": "file",
"target": "",
"options": {},
"package": None,
"custom": None,
})
for x in xrange(10, 101):
p.assert_any_call({
"id": x,
"category": "file",
"target": "",
"options": {},
"package": None,
"custom": None,
})
def init_binaries():
"""Inform the user about the need to periodically look for new analyzer
binaries. These include the Windows monitor etc."""
def throw():
raise CuckooStartupError(
"The binaries used for Windows analysis are updated regularly, "
"independently from the release line. It appears that you're "
"not up-to-date. This may happen when you've just installed the "
"latest development version of Cuckoo or when you've updated "
"to the latest Cuckoo. In order to get up-to-date, please run "
"the following command: `cuckoo community`."
)
dirpath = cwd("monitor", "latest")
# If "latest" is a symbolic link, check that it exists.
if os.path.islink(dirpath):
if not os.path.exists(dirpath):
throw()
# If "latest" is a file, check that it contains a legitimate hash.
elif os.path.isfile(dirpath):
monitor = os.path.basename(open(dirpath, "rb").read().strip())
if not monitor or not os.path.isdir(cwd("monitor", monitor)):
throw()
else:
throw()
def sqldump(dburi, dirpath):
args, env = dumpcmd(dburi, dirpath)
envargs = " ".join("%s=%s" % (k, v) for k, v in env.items())
cmdline = " ".join('"%s"' % arg if " " in arg else arg for arg in args)
cmd = "%s %s" % (envargs, cmdline) if envargs else cmdline
print "We can make a SQL database backup as follows:"
print "input cmd =>", cmd
print "output SQL =>", cwd("backup.sql")
if not click.confirm("Would you like to make a backup", default=True):
return
try:
subprocess.check_call(
args, stdout=open(cwd("backup.sql"), "wb"),
env=dict(os.environ.items() + env.items())
)
except (subprocess.CalledProcessError, OSError) as e:
raise CuckooOperationalError(
"Error creating SQL database dump as the command returned an "
"error code: %s. Please make sure that the required tooling "
"for making a database backup is installed and review the "
"database URI to make sure it's correct: %s!" % (e, dburi)
)
def test_monitor_latest_symlink(self):
set_cwd(tempfile.mktemp())
cuckoo_create()
monitor = open(cwd("monitor", "latest"), "rb").read().strip()
os.unlink(cwd("monitor", "latest"))
os.symlink(cwd("monitor", monitor), cwd("monitor", "latest"))
migrate_cwd()
def apply_template(cls):
# check elasticsearch version
es_version = elastic.info().get("version", {}).get('number', 'Invalid ES info schema')
# get the version 5 template
if es_version.startswith('5') or es_version.startswith('6'):
template_path = cwd("elasticsearch", "template-es5.json")
if not os.path.exists(template_path):
return False
elif es_version.startswith('2'):
template_path = cwd("elasticsearch", "template-es2.json")
if not os.path.exists(template_path):
return False
else:
raise CuckooReportError('Invalid Elasticsearch version %s unable to start the Elasticsearch'
'reporting module' % es_version)
try:
template = json.loads(open(template_path, "rb").read())
except ValueError:
raise CuckooReportError(
"Unable to read valid JSON from the ElasticSearch "
"template JSON file located at: %s" % template_path
)
# Create an index wildcard based off of the index name specified
# in the config file, this overwrites the settings in
# template.json.
template["template"] = elastic.index + "-*"
# if the template does not already exist then create it
if not elastic.client.indices.exists_template(cls.template_name):
elastic.client.indices.put_template(
name=cls.template_name, body=json.dumps(template)
)
return True
def test_process_json_logging():
set_cwd(tempfile.mkdtemp())
cuckoo_create()
init_yara()
init_logfile("process-p0.json")
def process_tasks(instance, maxcount):
logger("foo bar", action="hello.world", status="success")
with mock.patch("cuckoo.main.Database"):
with mock.patch("cuckoo.main.process_tasks") as p1:
with mock.patch("time.time") as p2:
p1.side_effect = process_tasks
p2.return_value = 1484232003
main.main(
("--cwd", cwd(), "process", "p0"), standalone_mode=False
)
assert json.load(open(cwd("log", "process-p0.json"), "rb")) == {
"asctime": mock.ANY,
"action": "hello.world",
"level": "info",
"message": "foo bar",
"status": "success",
"task_id": None,
"time": 1484232003,
}
def test_import_noconfirm(self, p):
set_cwd(tempfile.mkdtemp())
p.side_effect = True, False
dirpath = init_legacy_analyses()
os.makedirs(os.path.join(dirpath, "lib", "cuckoo", "common"))
open(os.path.join(
dirpath, "lib", "cuckoo", "common", "constants.py"
), "wb").write(constants_11_py)
shutil.copytree(
"tests/files/conf/110_plain", os.path.join(dirpath, "conf")
)
filepath = os.path.join(dirpath, "conf", "cuckoo.conf")
buf = open(filepath, "rb").read()
open(filepath, "wb").write(buf.replace(
"connection =", "connection = %s" % self.URI
))
main.main(
("--cwd", cwd(), "import", dirpath), standalone_mode=False
)
db = Database()
db.connect()
assert db.engine.name == self.ENGINE
assert open(cwd("logs", "a.txt", analysis=1), "rb").read() == "a"
assert config("cuckoo:database:connection") == self.URI
assert db.count_tasks() == 2
def test_migration_201_202():
set_cwd(tempfile.mkdtemp())
Folders.create(cwd(), "conf")
Files.create(cwd("conf"), "virtualbox.conf", """
[virtualbox]
machines = cuckoo1, cuckoo2
[cuckoo1]
platform = windows
[cuckoo2]
platform = windows
""")
# Except for virtualbox.
machineries = (
"avd", "esx", "kvm", "physical", "qemu",
"vmware", "vsphere", "xenserver",
)
for machinery in machineries:
Files.create(
cwd("conf"), "%s.conf" % machinery,
"[%s]\nmachines =" % machinery
)
cfg = Config.from_confdir(cwd("conf"), loose=True)
cfg = migrate(cfg, "2.0.1", "2.0.2")
assert cfg["virtualbox"]["cuckoo1"]["osprofile"] is None
assert cfg["virtualbox"]["cuckoo2"]["osprofile"] is None
def write_supervisor_conf(username):
"""Writes supervisord.conf configuration file if it does not exist yet."""
# TODO Handle updates?
if os.path.exists(cwd("supervisord.conf")):
return
if os.environ.get("VIRTUAL_ENV"):
virtualenv = os.path.join(os.environ["VIRTUAL_ENV"], "bin")
python_path = os.path.join(virtualenv, "python")
cuckoo_path = os.path.join(virtualenv, "cuckoo")
else:
python_path = "python"
cuckoo_path = "cuckoo"
template = jinja2.Environment().from_string(
open(cwd("cwd", "supervisord.jinja2", private=True), "rb").read()
)
with open(cwd("supervisord.conf"), "wb") as f:
f.write(template.render({
"cwd": cwd,
"username": username,
"cuckoo_path": cuckoo_path,
"python_path": python_path,
}).rstrip().encode("utf8") + "\n")
def test_migration_203_204():
set_cwd(tempfile.mkdtemp())
Folders.create(cwd(), "conf")
Files.create(cwd("conf"), "processing.conf", """
[dumptls]
enabled = on
""")
Files.create(cwd("conf"), "qemu.conf", """
[qemu]
machines = ubuntu32, ubuntu64
[ubuntu32]
arch = x86
[ubuntu64]
arch = x64
""")
cfg = Config.from_confdir(cwd("conf"), loose=True)
cfg = migrate(cfg, "2.0.3", "2.0.4")
assert cfg["processing"]["extracted"]["enabled"] is True
# Except for qemu.
machineries = (
"avd", "esx", "kvm", "physical", "virtualbox",
"vmware", "vsphere", "xenserver",
)
for machinery in machineries:
Files.create(
cwd("conf"), "%s.conf" % machinery, "[%s]\nmachines =" % machinery
)
assert cfg["qemu"]["ubuntu32"]["enable_kvm"] is False
assert cfg["qemu"]["ubuntu32"]["snapshot"] is None
def test_cuckoo_init_main(self):
"""Tests that 'cuckoo' works with a new CWD."""
main.main(
("--cwd", cwd(), "--nolog"),
standalone_mode=False
)
assert os.path.exists(os.path.join(cwd(), "mitm.py"))
def test_cuckoo_init_no_resultserver(self):
"""Tests that 'cuckoo init' doesn't launch the ResultServer."""
with pytest.raises(SystemExit):
main.main(
("--cwd", cwd(), "--nolog", "init"),
standalone_mode=False
)
# We copy the monitor binary directory over from user-CWD (which is
# also present in the Travis CI environment, etc) as otherwise the
# following call will raise an exception about not having found the
# monitoring binaries.
shutil.rmtree(os.path.join(cwd(), "monitor"))
shutil.copytree(
os.path.expanduser("~/.cuckoo/monitor"),
os.path.join(cwd(), "monitor")
)
# Raises CuckooCriticalError if ResultServer can't bind (which no
# longer happens now, naturally).
main.main(
("--cwd", cwd(), "--nolog", "init"),
standalone_mode=False
)
assert ResultServer not in Singleton._instances
def test_path(self, p):
set_cwd(tempfile.mkdtemp(prefix="."))
cuckoo_create()
mkdir(cwd(analysis=1))
def create():
mkdir(cwd("suricata", "files", analysis=1))
f = open(cwd("suricata", "files", "file.1", analysis=1), "wb")
f.write("a")
f = open(cwd("suricata", "eve.json", analysis=1), "wb")
f.write("")
f = open(cwd("suricata", "files-json.log", analysis=1), "wb")
f.write(json.dumps({
"id": 1,
"size": 1,
"filename": "a.txt",
}))
open(cwd("dump.pcap", analysis=1), "wb").write("pcap")
s = Suricata()
s.set_path(cwd(analysis=1))
s.set_options({})
s.process_pcap_binary = create
s.run()
def test_import_confirm(self, p):
set_cwd(tempfile.mkdtemp())
p.return_value = True
dirpath = init_legacy_analyses()
os.makedirs(os.path.join(dirpath, "lib", "cuckoo", "common"))
open(os.path.join(
dirpath, "lib", "cuckoo", "common", "constants.py"
), "wb").write(constants_11_py)
shutil.copytree(
"tests/files/conf/110_plain", os.path.join(dirpath, "conf")
)
filepath = os.path.join(dirpath, "conf", "cuckoo.conf")
buf = open(filepath, "rb").read()
open(filepath, "wb").write(buf.replace(
"connection =", "connection = %s" % self.URI
))
try:
main.main(
("--cwd", cwd(), "import", dirpath), standalone_mode=False
)
except CuckooOperationalError as e:
assert "SQL database dump as the command" in e.message
assert not is_linux()
return
db = Database()
db.connect()
assert db.engine.name == self.ENGINE
assert open(cwd("logs", "a.txt", analysis=1), "rb").read() == "a"
assert config("cuckoo:database:connection") == self.URI
assert db.count_tasks() == 2
def test_basics():
set_cwd(tempfile.mkdtemp())
cuckoo_create()
mkdir(cwd(analysis=1))
init_yara()
em = ExtractManager(1)
em.write_extracted("foo", "bar")
filepath = cwd("extracted", "0.foo", analysis=1)
assert open(filepath, "rb").read() == "bar"
scr = Scripting()
cmd = scr.parse_command(
"powershell -e %s" % "foobar".encode("utf-16le").encode("base64")
)
em.push_script({
"pid": 1,
"first_seen": 2,
}, cmd)
filepath = cwd("extracted", "0.ps1", analysis=1)
assert open(filepath, "rb").read() == "foobar"
em.push_command_line(
"powershell -e %s" % "world!".encode("utf-16le").encode("base64")
)
filepath = cwd("extracted", "1.ps1", analysis=1)
assert open(filepath, "rb").read() == "world!"
def test_empty_move(self):
oldfilepath = Files.temp_put("hello")
movesql("sqlite:///%s" % oldfilepath, "move", temppath())
assert not os.path.exists(oldfilepath)
assert os.path.exists(cwd("cuckoo.db"))
assert not os.path.islink(cwd("cuckoo.db"))
assert open(cwd("cuckoo.db"), "rb").read() == "hello"
def test_load_signatures():
set_cwd(tempfile.mkdtemp())
cuckoo_create()
shutil.rmtree(cwd("signatures"))
shutil.copytree("tests/files/enumplugins", cwd("signatures"))
sys.modules.pop("signatures", None)
load_signatures()
# Ensure that the Signatures are loaded in the global list.
names = []
for sig in cuckoo.signatures:
names.append(sig.__module__)
assert "signatures.sig1" in names
assert "signatures.sig2" in names
assert "signatures.sig3" in names
# Ensure that the Signatures are loaded in the RunSignatures object.
RunSignatures.init_once()
rs, names = RunSignatures({}), []
for sig in rs.signatures:
names.append(sig.__class__.__name__)
assert "Sig1" in names
assert "Sig2" in names
assert "Sig3" in names
def migrate(cls):
tasks = cls.d.engine.execute(
"SELECT status FROM tasks ORDER BY id"
).fetchall()
assert tasks[0][0] == "success"
assert tasks[1][0] == "processing"
assert tasks[2][0] == "pending"
main.main(
("--cwd", cwd(), "migrate", "--revision", "263a45963c72"),
standalone_mode=False
)
tasks = cls.d.engine.execute(
"SELECT status FROM tasks ORDER BY id"
).fetchall()
assert tasks[0][0] == "completed"
assert tasks[1][0] == "running"
assert tasks[2][0] == "pending"
main.main(
("--cwd", cwd(), "migrate"),
standalone_mode=False
)
tasks = cls.d.engine.execute(
"SELECT status, owner FROM tasks ORDER BY id"
).fetchall()
assert tasks[0][0] == "completed"
assert tasks[0][1] is None
assert tasks[1][0] == "running"
assert tasks[2][0] == "pending"
def tasks_report(task_id, report_format="json"):
formats = {
"json": "report.json",
"html": "report.html",
"maec": "report.MAEC-5.0.json",
}
bz_formats = {
"all": {"type": "-", "files": ["memory.dmp"]},
"dropped": {"type": "+", "files": ["files"]},
"package_files": {"type": "+", "files": ["package_files"]},
}
tar_formats = {
"bz2": "w:bz2",
"gz": "w:gz",
"tar": "w",
}
if report_format.lower() in formats:
report_path = cwd(
"storage", "analyses", "%d" % task_id, "reports",
formats[report_format.lower()]
)
elif report_format.lower() in bz_formats:
bzf = bz_formats[report_format.lower()]
srcdir = cwd("storage", "analyses", "%d" % task_id)
s = io.BytesIO()
# By default go for bz2 encoded tar files (for legacy reasons).
tarmode = tar_formats.get(request.args.get("tar"), "w:bz2")
tar = tarfile.open(fileobj=s, mode=tarmode, dereference=True)
for filedir in os.listdir(srcdir):
filepath = os.path.join(srcdir, filedir)
if not os.path.exists(filepath):
continue
if bzf["type"] == "-" and filedir not in bzf["files"]:
tar.add(filepath, arcname=filedir)
if bzf["type"] == "+" and filedir in bzf["files"]:
tar.add(filepath, arcname=filedir)
tar.close()
response = make_response(s.getvalue())
response.headers["Content-Type"] = \
"application/x-tar; charset=UTF-8"
return response
else:
return json_error(400, "Invalid report format")
if not os.path.exists(report_path):
return json_error(404, "Report not found")
if report_format == "json" or report_format == "maec":
response = make_response(open(report_path, "rb").read())
response.headers["Content-Type"] = "application/json"
return response
else:
return open(report_path, "rb").read()
def init_yara():
"""Initialize & load/compile Yara rules."""
categories = (
"binaries", "urls", "memory", "scripts", "shellcode", "dumpmem",
)
log.debug("Initializing Yara...")
for category in categories:
dirpath = cwd("yara", category)
if not os.path.exists(dirpath):
log.warning("Missing Yara directory: %s?", dirpath)
rules, indexed = {}, []
for dirpath, dirnames, filenames in os.walk(dirpath, followlinks=True):
for filename in filenames:
if not filename.endswith((".yar", ".yara")):
continue
filepath = os.path.join(dirpath, filename)
try:
# TODO Once Yara obtains proper Unicode filepath support we
# can remove this check. See also this Github issue:
# https://github.com/VirusTotal/yara-python/issues/48
assert len(str(filepath)) == len(filepath)
except (UnicodeEncodeError, AssertionError):
log.warning(
"Can't load Yara rules at %r as Unicode filepaths are "
"currently not supported in combination with Yara!",
filepath
)
continue
rules["rule_%s_%d" % (category, len(rules))] = filepath
indexed.append(filename)
try:
File.yara_rules[category] = yara.compile(filepaths=rules)
except yara.Error as e:
raise CuckooStartupError(
"There was a syntax error in one or more Yara rules: %s" % e
)
# The memory.py processing module requires a yara file with all of its
# rules embedded in it, so create this file to remain compatible.
if category == "memory":
f = open(cwd("stuff", "index_memory.yar"), "wb")
for filename in indexed:
f.write('include "%s"\n' % cwd("yara", "memory", filename))
indexed = sorted(indexed)
for entry in indexed:
if (category, entry) == indexed[-1]:
log.debug("\t `-- %s %s", category, entry)
else:
log.debug("\t |-- %s %s", category, entry)
# Store the compiled Yara rules for the "dumpmem" category in
# $CWD/stuff/ so that we may pass it along to zer0m0n during analysis.
File.yara_rules["dumpmem"].save(cwd("stuff", "dumpmem.yarac"))
def fetch_community(branch="master", force=False, filepath=None):
if filepath:
buf = open(filepath, "rb").read()
else:
log.info("Downloading.. %s", URL % branch)
r = requests.get(URL % branch)
if r.status_code != 200:
raise CuckooOperationalError(
"Error fetching the Cuckoo Community binaries "
"(status_code: %d)!" % r.status_code
)
buf = r.content
t = tarfile.TarFile.open(fileobj=io.BytesIO(buf), mode="r:gz")
folders = {
"modules/signatures": "signatures",
"data/monitor": "monitor",
"data/yara": "yara",
"agent": "agent",
"analyzer": "analyzer",
}
members = t.getmembers()
directory = members[0].name.split("/")[0]
for tarfolder, outfolder in folders.items():
mkdir(cwd(outfolder))
# E.g., "community-master/modules/signatures".
name_start = "%s/%s" % (directory, tarfolder)
for member in members:
if not member.name.startswith(name_start) or \
name_start == member.name:
continue
filepath = cwd(outfolder, member.name[len(name_start)+1:])
if member.isdir():
mkdir(filepath)
continue
# TODO Ask for confirmation as we used to do.
if os.path.exists(filepath) and not force:
log.debug(
"Not overwriting file which already exists: %s",
member.name[len(name_start)+1:]
)
continue
if member.issym():
t.makelink(member, filepath)
continue
if not os.path.exists(os.path.dirname(filepath)):
os.makedirs(os.path.dirname(filepath))
log.debug("Extracted %s..", member.name[len(name_start)+1:])
open(filepath, "wb").write(t.extractfile(member).read())
def test_main(self, p, q):
p.side_effect = SystemExit(0)
# Ensure that the "latest" binary value makes sense so that the
# "run community command" exception is not thrown.
mkdir(cwd("monitor", open(cwd("monitor", "latest")).read().strip()))
main.main(("--cwd", cwd(), "-d", "--nolog"), standalone_mode=False)
q.assert_called_once()
def test_symlink(self):
if not is_linux():
return
# Include all Yara rules from binaries/ into memory/ as well.
os.symlink(cwd("yara", "binaries"), cwd("yara", "memory", "bins"))
init_yara()
assert len(list(File.yara_rules["memory"])) == 5
def test_cuckoo_init_main_nosigs(self, p):
"""Ensure load_signatures() isn't called for 'cuckoo' with new CWD."""
main.main(
("--cwd", cwd(), "--nolog"),
standalone_mode=False
)
assert os.path.exists(os.path.join(cwd(), "mitm.py"))
p.assert_not_called()
def test_remove_pidfile():
set_cwd(tempfile.mkdtemp())
cuckoo_create()
Pidfile("test2").create()
assert os.path.exists(cwd("pidfiles", "test2.pid"))
Pidfile("test2").remove()
assert not os.path.exists(cwd("pidfiles", "test2.pid"))
def migrate_database(revision="head"):
args = [
"alembic", "-x", "cwd=%s" % cwd(), "upgrade", revision,
]
try:
subprocess.check_call(args, cwd=cwd("db_migration", private=True))
except subprocess.CalledProcessError:
return False
return True
def create_app():
app = flask.Flask(
"Distributed Cuckoo",
template_folder=cwd("..", "distributed", "templates", private=True),
static_folder=cwd("..", "distributed", "static", private=True),
)
init_settings()
app.config.from_object(settings)
for blueprint, routes in blueprints:
for route in routes:
app.register_blueprint(blueprint, url_prefix=route)
db.init_app(app)
db.create_all(app=app)
# Check whether an alembic version is present and whether
# we're up-to-date.
with app.app_context():
row = AlembicVersion.query.first()
if not row:
db.session.add(AlembicVersion(AlembicVersion.VERSION))
db.session.commit()
elif row.version_num != AlembicVersion.VERSION:
sys.exit("Your database is not up-to-date, please upgrade it "
"(run `cuckoo distributed migrate`).")
# Further check the configuration.
if not settings.SQLALCHEMY_DATABASE_URI:
sys.exit("Please configure a database connection.")
if not settings.report_formats:
sys.exit("Please configure one or more reporting formats.")
if not settings.samples_directory or \
not os.path.isdir(settings.samples_directory):
sys.exit("Please configure a samples directory path.")
if not settings.reports_directory or \
not os.path.isdir(settings.reports_directory):
sys.exit("Please configure a reports directory path.")
@app.after_request
def custom_headers(response):
"""Set some custom headers across all HTTP responses."""
response.headers["Server"] = "Distributed Machete Server"
response.headers["X-Content-Type-Options"] = "nosniff"
response.headers["X-Frame-Options"] = "DENY"
response.headers["X-XSS-Protection"] = "1; mode=block"
response.headers["Pragma"] = "no-cache"
response.headers["Cache-Control"] = "no-cache"
response.headers["Expires"] = "0"
return response
return app
def analyzer_zipfile(platform, monitor):
"""Creates the Zip file that is sent to the Guest."""
t = time.time()
zip_data = io.BytesIO()
zip_file = zipfile.ZipFile(zip_data, "w", zipfile.ZIP_STORED)
# Select the proper analyzer's folder according to the operating
# system associated with the current machine.
root = cwd("analyzer", platform)
root_len = len(os.path.abspath(root))
if not os.path.exists(root):
log.error("No valid analyzer found at path: %s", root)
raise CuckooGuestError(
"No valid analyzer found for %s platform!" % platform
)
# Walk through everything inside the analyzer's folder and write
# them to the zip archive.
for root, dirs, files in os.walk(root):
archive_root = os.path.abspath(root)[root_len:]
for name in files:
path = os.path.join(root, name)
archive_name = os.path.join(archive_root, name)
zip_file.write(path, archive_name)
# Include the chosen monitoring component and any additional files.
if platform == "windows":
dirpath = cwd("monitor", monitor)
# Generally speaking we should no longer be getting symbolic links for
# "latest" anymore, so in the case of a file; follow it.
if os.path.isfile(dirpath):
monitor = os.path.basename(open(dirpath, "rb").read().strip())
dirpath = cwd("monitor", monitor)
for name in os.listdir(dirpath):
zip_file.write(
os.path.join(dirpath, name), os.path.join("bin", name)
)
# Dump compiled "dumpmem" Yara rules for zer0m0n usage.
zip_file.write(cwd("stuff", "dumpmem.yarac"), "bin/rules.yarac")
zip_file.close()
data = zip_data.getvalue()
if time.time() - t > 10:
log.warning(
"It took more than 10 seconds to build the Analyzer Zip for the "
"Guest. This might be a serious performance penalty. Is your "
"analyzer/windows/ directory bloated with unnecessary files?"
)
return data
def test_dnsserve_abort(self, p, capsys):
p.side_effect = KeyboardInterrupt
main.main(("--cwd", cwd(), "dnsserve"), standalone_mode=False)
out, _ = capsys.readouterr()
assert "Aborting Cuckoo DNS Serve" in out
def cuckoo_clean():
"""Clean up cuckoo setup.
It deletes logs, all stored data from file system and configured
databases (SQL and MongoDB).
"""
# Init logging (without writing to file).
init_console_logging()
try:
# Initialize the database connection.
db = Database()
db.connect(schema_check=False)
# Drop all tables.
db.drop()
except (CuckooDependencyError, CuckooDatabaseError) as e:
# If something is screwed due to incorrect database migrations or bad
# database SqlAlchemy would be unable to connect and operate.
log.warning(
"Error connecting to database: it is suggested to check "
"the connectivity, apply all migrations if needed or purge "
"it manually. Error description: %s", e)
# Check if MongoDB reporting is enabled and drop the database if it is.
if mongo.init():
try:
mongo.connect()
mongo.drop()
mongo.close()
except Exception as e:
log.warning("Unable to drop MongoDB database: %s", e)
# Check if ElasticSearch reporting is enabled and drop its data if it is.
if elastic.init():
elastic.connect()
# TODO This should be moved to the elastic abstract.
# TODO We should also drop historic data, i.e., from pervious days,
# months, and years.
date_index = datetime.datetime.utcnow().strftime({
"yearly": "%Y",
"monthly": "%Y-%m",
"daily": "%Y-%m-%d",
}[elastic.index_time_pattern])
dated_index = "%s-%s" % (elastic.index, date_index)
elastic.client.indices.delete(index=dated_index, ignore=[400, 404])
template_name = "%s_template" % dated_index
if elastic.client.indices.exists_template(template_name):
elastic.client.indices.delete_template(template_name)
# Paths to clean.
paths = [
cwd("cuckoo.db"),
cwd("log"),
cwd("storage", "analyses"),
cwd("storage", "baseline"),
cwd("storage", "binaries"),
]
# Delete the various files and directories. In case of directories, keep
# the parent directories, so to keep the state of the CWD in tact.
for path in paths:
if os.path.isdir(path):
try:
shutil.rmtree(path)
os.mkdir(path)
except (IOError, OSError) as e:
log.warning("Error removing directory %s: %s", path, e)
elif os.path.isfile(path):
try:
os.unlink(path)
except (IOError, OSError) as e:
log.warning("Error removing file %s: %s", path, e)
def test_clean(self):
with mock.patch("cuckoo.main.cuckoo_clean") as p:
p.return_value = None
main.main(("--cwd", cwd(), "clean"), standalone_mode=False)
p.assert_called_once_with()
def test_process_once(self, p, q):
main.main(("--cwd", cwd(), "process", "-r", "1234"),
standalone_mode=False)
p.assert_called_once_with("1234")
q.assert_called_once()
def _build_whitelist(self):
result = []
whitelist_path = cwd("whitelist", "domain.txt", private=True)
for line in open(whitelist_path, "rb"):
result.append(line.strip())
return result
def test_clean_keepdirs(p):
set_cwd(tempfile.mkdtemp())
cuckoo_create()
with open(cwd("log", "cuckoo.log"), "wb") as f:
f.write("this is a log file")
os.mkdir(cwd(analysis=1))
with open(cwd("analysis.log", analysis=1), "wb") as f:
f.write("this is also a log file")
with open(cwd("storage", "binaries", "a"*40), "wb") as f:
f.write("this is a binary file")
assert os.path.isdir(cwd("log"))
assert os.path.exists(cwd("log", "cuckoo.log"))
assert os.path.exists(cwd("storage", "analyses"))
assert os.path.exists(cwd("storage", "analyses", "1"))
assert os.path.exists(cwd("storage", "analyses", "1", "analysis.log"))
assert os.path.exists(cwd("storage", "baseline"))
assert os.path.exists(cwd("storage", "binaries"))
assert os.path.exists(cwd("storage", "binaries", "a"*40))
cuckoo_clean()
assert os.path.isdir(cwd("log"))
assert not os.path.exists(cwd("log", "cuckoo.log"))
assert os.path.exists(cwd("storage", "analyses"))
assert not os.path.exists(cwd("storage", "analyses", "1"))
assert not os.path.exists(cwd("storage", "analyses", "1", "analysis.log"))
assert os.path.exists(cwd("storage", "baseline"))
assert os.path.exists(cwd("storage", "binaries"))
assert not os.path.exists(cwd("storage", "binaries", "a"*40))
def init_yara():
"""Initialize & load/compile Yara rules."""
categories = (
"binaries",
"urls",
"memory",
"scripts",
"shellcode",
"office",
)
log.debug("Initializing Yara...")
for category in categories:
dirpath = cwd("yara", category)
if not os.path.exists(dirpath):
log.warning("Missing Yara directory: %s?", dirpath)
rules, indexed = {}, []
for dirpath, dirnames, filenames in os.walk(dirpath, followlinks=True):
for filename in filenames:
if not filename.endswith((".yar", ".yara")):
continue
filepath = os.path.join(dirpath, filename)
try:
# TODO Once Yara obtains proper Unicode filepath support we
# can remove this check. See also this Github issue:
# https://github.com/VirusTotal/yara-python/issues/48
assert len(str(filepath)) == len(filepath)
except (UnicodeEncodeError, AssertionError):
log.warning(
"Can't load Yara rules at %r as Unicode filepaths are "
"currently not supported in combination with Yara!",
filepath)
continue
rules["rule_%s_%d" % (category, len(rules))] = filepath
indexed.append(filename)
# Need to define each external variable that will be used in the
# future. Otherwise Yara will complain.
externals = {
"filename": "",
}
try:
File.yara_rules[category] = yara.compile(filepaths=rules,
externals=externals)
except yara.Error as e:
raise CuckooStartupError(
"There was a syntax error in one or more Yara rules: %s" % e)
# The memory.py processing module requires a yara file with all of its
# rules embedded in it, so create this file to remain compatible.
if category == "memory":
f = open(cwd("stuff", "index_memory.yar"), "wb")
for filename in indexed:
f.write('include "%s"\n' % cwd("yara", "memory", filename))
indexed = sorted(indexed)
for entry in indexed:
if (category, entry) == indexed[-1]:
log.debug("\t `-- %s %s", category, entry)
else:
log.debug("\t |-- %s %s", category, entry)
# Store the compiled Yara rules for the "memory" category in $CWD/stuff/
# so that we may easily pass it along to zer0m0n during an analysis.
File.yara_rules["memory"].save(cwd("stuff", "dumpmem.yarac"))
def test_rooter_abort(self, p, capsys):
p.side_effect = KeyboardInterrupt
main.main(("--cwd", cwd(), "rooter"), standalone_mode=False)
out, _ = capsys.readouterr()
assert "Aborting the Cuckoo Rooter" in out
def test_api(self):
with mock.patch("cuckoo.main.cuckoo_api") as p:
p.return_value = None
main.main(("--cwd", cwd(), "api"), standalone_mode=False)
p.assert_called_once_with("localhost", 8090, False)
def test_community_abort(self, p, capsys):
p.side_effect = KeyboardInterrupt
main.main(("--cwd", cwd(), "community"), standalone_mode=False)
out, _ = capsys.readouterr()
assert "Aborting fetching of" in out
def test_cuckoo_conf(self):
Folders.create(cwd(), "conf")
write_cuckoo_conf()
def __init__(self, task, results):
"""@param analysis_path: analysis folder path."""
self.task = task
self.results = results
self.analysis_path = cwd("storage", "analyses", "%s" % task["id"])
def test_dnsserve(self):
with mock.patch("cuckoo.main.cuckoo_dnsserve") as p:
p.return_value = None
main.main(("--cwd", cwd(), "dnsserve"), standalone_mode=False)
p.assert_called_once_with("0.0.0.0", 53, None, None)
def test_process_init_modules(self, p, q):
main.main(
("--cwd", cwd(), "process", "-r", "1"),
standalone_mode=False
)
p.assert_called_once()
workers[tn] = None
time.sleep(5)
if os.environ.get("CUCKOO_APP") == "worker":
decide_cwd(exists=True)
if not HAVE_GEVENT:
sys.exit("Please install Distributed Cuckoo dependencies (through "
"`pip install cuckoo[distributed]`)")
formatter = logging.Formatter(
"%(asctime)s [%(name)s] %(levelname)s: %(message)s")
fh = logging.handlers.WatchedFileHandler(cwd("log", "distributed.log"))
fh.setFormatter(formatter)
logging.getLogger().addHandler(fh)
# Create the Flask object and push its context so that we can reuse the
# database connection throughout our script.
app = create_app()
workers = {
("dist.scheduler", True):
gevent.spawn(with_app, "dist.scheduler", scheduler),
("dist.status", True):
gevent.spawn(with_app, "dist.status", scheduler),
}
with_app("dist.spawner", spawner)
def test_process_task_range_multi_db(self, p):
mkdir(cwd(analysis=1234))
mkdir(cwd(analysis=2345))
p.return_value.view_task.return_value = {}
process_task_range("2345")
p.return_value.view_task.assert_called_once_with(2345)
def test_process_many(self, p, q):
main.main(("--cwd", cwd(), "process", "instance"),
standalone_mode=False)
p.assert_called_once_with("instance", 0)
q.assert_called_once()
def import_(self, f, submit_id):
"""Import an analysis identified by the file(-like) object f."""
try:
z = zipfile.ZipFile(f)
except zipfile.BadZipfile:
raise CuckooOperationalError(
"Imported analysis is not a proper .zip file.")
# Ensure there are no files with illegal or potentially insecure names.
# TODO Keep in mind that if we start to support other archive formats
# (e.g., .tar) that those may also support symbolic links. In that case
# we should probably start using sflock here.
for filename in z.namelist():
if filename.startswith("/") or ".." in filename or ":" in filename:
raise CuckooOperationalError(
"The .zip file contains a file with a potentially "
"incorrect filename: %s" % filename)
if "task.json" not in z.namelist():
raise CuckooOperationalError(
"The task.json file is required in order to be able to import "
"an analysis! This file contains metadata about the analysis.")
required_fields = {
"options": dict,
"route": basestring,
"package": basestring,
"target": basestring,
"category": basestring,
"memory": bool,
"timeout": (int, long),
"priority": (int, long),
"custom": basestring,
"tags": (tuple, list),
}
try:
info = json.loads(z.read("task.json"))
for key, type_ in required_fields.items():
if key not in info:
raise ValueError("missing %s" % key)
if info[key] is not None and not isinstance(info[key], type_):
raise ValueError("%s => %s" % (key, info[key]))
except ValueError as e:
raise CuckooOperationalError(
"The provided task.json file, required for properly importing "
"the analysis, is incorrect or incomplete (%s)." % e)
if info["category"] == "url":
task_id = db.add_url(url=info["target"],
package=info["package"],
timeout=info["timeout"],
options=info["options"],
priority=info["priority"],
custom=info["custom"],
memory=info["memory"],
tags=info["tags"],
submit_id=submit_id)
else:
# Users may have the "delete_bin_copy" enabled and in such cases
# the binary file won't be included in the .zip file.
if "binary" in z.namelist():
filepath = Files.temp_named_put(
z.read("binary"), os.path.basename(info["target"]))
else:
filepath = __file__
# We'll be updating the target shortly.
task_id = db.add_path(file_path=filepath,
package=info["package"],
timeout=info["timeout"],
options=info["options"],
priority=info["priority"],
custom=info["custom"],
memory=info["memory"],
tags=info["tags"],
submit_id=submit_id)
if not task_id:
raise CuckooOperationalError(
"There was an error creating a task for the to-be imported "
"analysis in our database.. Can't proceed.")
# The constructors currently don't accept this argument.
db.set_route(task_id, info["route"])
mkdir(cwd(analysis=task_id))
z.extractall(cwd(analysis=task_id))
# If there's an analysis.json file, load it up to figure out additional
# metdata regarding this analysis.
if os.path.exists(cwd("analysis.json", analysis=task_id)):
try:
obj = json.load(
open(cwd("analysis.json", analysis=task_id), "rb"))
if not isinstance(obj, dict):
raise ValueError
if "errors" in obj and not isinstance(obj["errors"], list):
raise ValueError
if "action" in obj and not isinstance(obj["action"], list):
raise ValueError
except ValueError:
log.warning(
"An analysis.json file was provided, but wasn't a valid "
"JSON object/structure that we can to enhance the "
"analysis information.")
else:
for error in set(obj.get("errors", [])):
if isinstance(error, basestring):
db.add_error(error, task_id)
for action in set(obj.get("action", [])):
if isinstance(action, basestring):
db.add_error("", task_id, action)
# We set this analysis as completed so that it will be processed
# automatically (assuming 'cuckoo process' is running).
db.set_status(task_id, TASK_COMPLETED)
return task_id
def test_init(p):
set_cwd(tempfile.mkdtemp())
with pytest.raises(SystemExit):
main.main(("--cwd", cwd(), "--nolog", "init"), standalone_mode=False)
p.assert_not_called()
def start(self, label, task):
"""Start a virtual machine.
@param label: virtual machine name.
@param task: task object.
"""
log.debug("Starting vm %s", label)
vmname = self.db.view_machine_by_label(label).name
vm_state_timeout = config("cuckoo:timeouts:vm_state")
try:
args = [
"sudo", self.options.avd.emulator_path,
"@%s" % label,
"-no-snapshot-save",
"-net-tap", "tap_%s" % vmname,
"-net-tap-script-up", cwd("stuff", "setup-hostnet-avd.sh")
]
# Aggregate machine-specific options.
for machine in self.machines():
if machine.label == label:
# In headless mode we remove the audio, and window support.
if "headless" in machine.options:
args += ["-no-audio", "-no-window"]
# Retrieve the snapshot name for this machine to load it.
args += ["-snapshot", machine.snapshot]
break
# Create a socket server to acquire the console port of the emulator.
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setblocking(0)
s.bind(("127.0.0.1", 0))
s.listen(0)
args += ["-report-console", "tcp:%s" % s.getsockname()[1]]
# Start the emulator process..
emu_conn = None
proc = subprocess.Popen(
args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
rlist = [s]
time_cnt = 0
while True:
if proc.poll() is not None:
out, err = proc.communicate()
# Grab emulator error message from stderr & stdout
exc_info = ""
for line in out.splitlines():
if "emulator: ERROR: " in line:
exc_info += "%s\n" % line
exc_info += err
raise OSError(exc_info.rstrip())
rds, _, _ = select.select(rlist, [], [], 0)
sock2read = rds.pop() if rds else None
if sock2read == s:
emu_conn, _ = s.accept()
emu_conn.setblocking(0)
rlist[0] = emu_conn
elif emu_conn and sock2read == emu_conn:
emu_port = emu_conn.recv(1024)
break
if time_cnt < vm_state_timeout:
time.sleep(1)
time_cnt += 1
else:
proc.terminate()
raise OSError("timed out")
self._emulator_labels[label] = "emulator-%s" % emu_port
except OSError as e:
raise CuckooMachineError(
"Emulator failed starting machine %s: %s" % (label, e)
)
except IOError as e:
raise CuckooMachineError(e)
finally:
s.close()
if emu_conn:
emu_conn.close()
self._wait_status_ready(label)
def test_process_abort(self, p, q, capsys):
p.side_effect = KeyboardInterrupt
main.main(("--cwd", cwd(), "process", "-r", "1234"),
standalone_mode=False)
out, _ = capsys.readouterr()
assert "Aborting (re-)processing of your analyses" in out
def web(ctx, args, host, port, uwsgi, nginx):
"""Operate the Cuckoo Web Interface.
Use "--help" to get this help message and "help" to find Django's
manage.py potential subcommands.
"""
username = ctx.parent.user or getuser()
if uwsgi:
print "[uwsgi]"
print "plugins = python"
if os.environ.get("VIRTUAL_ENV"):
print "virtualenv =", os.environ["VIRTUAL_ENV"]
print "module = cuckoo.web.web.wsgi"
print "uid =", username
print "gid =", username
dirpath = os.path.join(cuckoo.__path__[0], "web", "static")
print "static-map = /static=%s" % dirpath
print "# If you're getting errors about the PYTHON_EGG_CACHE, then"
print "# uncomment the following line and add some path that is"
print "# writable from the defined user."
print "# env = PYTHON_EGG_CACHE="
print "env = CUCKOO_APP=web"
print "env = CUCKOO_CWD=%s" % cwd()
return
if nginx:
print "upstream _uwsgi_cuckoo_web {"
print " server unix:/run/uwsgi/app/cuckoo-web/socket;"
print "}"
print
print "server {"
print " listen %s:%d;" % (host, port)
print
print " # Cuckoo Web Interface"
print " location / {"
print " client_max_body_size 1G;"
print " proxy_redirect off;"
print " proxy_set_header X-Forwarded-Proto $scheme;"
print " uwsgi_pass _uwsgi_cuckoo_web;"
print " include uwsgi_params;"
print " }"
print "}"
return
# Switch to cuckoo/web and add the current path to sys.path as the Web
# Interface is using local imports here and there.
# TODO Rename local imports to either cuckoo.web.* or relative imports.
os.chdir(os.path.join(cuckoo.__path__[0], "web"))
sys.path.insert(0, ".")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cuckoo.web.web.settings")
# The Django HTTP server also imports the WSGI module for some reason, so
# ensure that WSGI is able to load.
os.environ["CUCKOO_APP"] = "web"
os.environ["CUCKOO_CWD"] = cwd()
from django.core.management import execute_from_command_line
init_console_logging(level=ctx.parent.level)
Database().connect()
if not args:
execute_from_command_line(
("cuckoo", "runserver", "%s:%d" % (host, port)))
else:
execute_from_command_line(("cuckoo", ) + args)
def test_dist_instance(self):
with mock.patch("cuckoo.main.cuckoo_distributed_instance") as p:
p.return_value = None
main.main(("--cwd", cwd(), "distributed", "instance", "name"),
standalone_mode=False)
p.assert_called_once_with("name")
def __init__(self, file_name="cuckoo", cfg=None, strict=False,
loose=False, raw=False):
"""
@param file_name: file name without extension.
@param cfg: configuration file path.
"""
env = {}
for key, value in os.environ.items():
if key.startswith("CUCKOO_"):
env[key] = value
env["CUCKOO_CWD"] = cwd()
env["CUCKOO_APP"] = os.environ.get("CUCKOO_APP", "")
config = ConfigParser.ConfigParser(env)
self.env_keys = []
for key in env.keys():
self.env_keys.append(key.lower())
self.sections = {}
try:
config.read(cfg or cwd("conf", "%s.conf" % file_name))
except ConfigParser.ParsingError as e:
raise CuckooConfigurationError(
"There was an error reading in the $CWD/conf/%s.conf "
"configuration file. Most likely there are leading "
"whitespaces in front of one of the key=value lines defined. "
"More information from the original exception: %s" %
(file_name, e)
)
if file_name not in self.configuration and not loose:
log.error("Unknown config file %s.conf", file_name)
return
for section in config.sections():
types = self.get_section_types(file_name, section, strict, loose)
if types is None:
continue
self.sections[section] = Dictionary()
setattr(self, section, self.sections[section])
try:
items = config.items(section)
except ConfigParser.InterpolationMissingOptionError as e:
log.error("Missing environment variable(s): %s", e)
raise CuckooConfigurationError(
"Missing environment variable: %s" % e
)
for name, raw_value in items:
if name in self.env_keys:
continue
if "\n" in raw_value:
wrong_key = "???"
try:
wrong_key = raw_value.split("\n", 1)[1].split()[0]
except:
pass
raise CuckooConfigurationError(
"There was an error reading in the $CWD/conf/%s.conf "
"configuration file. Namely, there are one or more "
"leading whitespaces before the definition of the "
"'%s' key/value pair in the '%s' section. Please "
"remove those leading whitespaces as Python's default "
"configuration parser is unable to handle those "
"properly." % (file_name, wrong_key, section)
)
if not raw and name in types:
# TODO Is this the area where we should be checking the
# configuration values?
# if not types[name].check(raw_value):
# print file_name, section, name, raw_value
# raise
value = types[name].parse(raw_value)
else:
if not loose:
log.error(
"Type of config parameter %s:%s:%s not found! "
"This may indicate that you've incorrectly filled "
"out the Cuckoo configuration, please double "
"check it.", file_name, section, name
)
value = raw_value
self.sections[section][name] = value
def test_dist_server(self):
with mock.patch("cuckoo.main.cuckoo_distributed") as p:
p.return_value = None
main.main(("--cwd", cwd(), "distributed", "server"),
standalone_mode=False)
p.assert_called_once_with("localhost", 9003, False)
def test_clean_abort(self, p, capsys):
p.side_effect = KeyboardInterrupt
main.main(("--cwd", cwd(), "clean"), standalone_mode=False)
out, _ = capsys.readouterr()
assert "Aborting cleaning up of" in out
def test_process_task_range_single_db(self, p):
mkdir(cwd(analysis=1234))
p.return_value.view_task.return_value = {}
process_task_range("1234")
p.return_value.view_task.assert_called_once_with(1234)
def migrate_cwd():
log.warning(
"This is the first time you're running Cuckoo after updating your "
"local version of Cuckoo. We're going to update files in your CWD "
"that require updating. Note that we'll first ensure that no custom "
"patches have been applied by you before applying any modifications "
"of our own.")
# Remove now-obsolete index_*.yar files.
for filename in os.listdir(cwd("yara")):
if filename.startswith("index_") and filename.endswith(".yar"):
os.remove(cwd("yara", filename))
# Create the new $CWD/stuff/ directory.
if not os.path.exists(cwd("stuff")):
mkdir(cwd("stuff"))
# Create the new $CWD/yara/dumpmem/ directory.
if not os.path.exists(cwd("yara", "dumpmem")):
mkdir(cwd("yara", "dumpmem"))
hashes = {}
for line in open(cwd("cwd", "hashes.txt", private=True), "rb"):
if not line.strip() or line.startswith("#"):
continue
hash_, filename = line.split()
hashes[filename] = hashes.get(filename, []) + [hash_]
modified, outdated = [], []
for filename, hashes in hashes.items():
if not os.path.exists(cwd(filename)):
outdated.append(filename)
continue
hash_ = hashlib.sha1(open(cwd(filename), "rb").read()).hexdigest()
if hash_ not in hashes:
modified.append(filename)
if hash_ != hashes[-1]:
outdated.append(filename)
if modified:
log.error(
"One or more files in the CWD have been modified outside of "
"regular Cuckoo usage. Due to these changes Cuckoo isn't able to "
"automatically upgrade your setup.")
for filename in sorted(modified):
log.warning("Modified file: %s (=> %s)", filename, cwd(filename))
log.error("Moving forward you have two options:")
log.warning(
"1) You make a backup of the affected files, remove their "
"presence in the CWD (yes, actually 'rm -f' the file), and "
"re-run Cuckoo to automatically restore the new version of the "
"file. Afterwards you'll be able to re-apply any changes as you "
"like.")
log.warning(
"2) You revert back to the version of Cuckoo you were on "
"previously and accept that manual changes that have not been "
"merged upstream require additional maintenance that you'll "
"pick up at a later point in time.")
sys.exit(1)
for filename in outdated:
log.debug("Upgraded %s", filename)
if not os.path.exists(os.path.dirname(cwd(filename))):
os.makedirs(os.path.dirname(cwd(filename)))
shutil.copy(cwd("..", "data", filename, private=True), cwd(filename))
log.info("Automated migration of your CWD was successful! Continuing "
"execution of Cuckoo as expected.")
def __init__(self, task):
"""@param task: task dictionary of the analysis to process."""
self.task = task
self.machine = {}
self.analysis_path = cwd(analysis=task["id"])
self.baseline_path = cwd("storage", "baseline")
def test_submit(self):
with mock.patch("cuckoo.main.submit_tasks") as p:
p.return_value = []
main.main((
"--cwd", cwd(), "submit", Files.create(cwd(), "a.txt", "hello")
), standalone_mode=False)