def finalize(self, db):
"""Executed by the scheduler when the analysis manager thread exists.
Updates the task status to the correct one and updates the
task.json."""
self.task.set_latest()
self.release_machine_lock()
# If, at this point, the analysis is not stopped, it cannot
# succeeded, since the manager thread already exited. Updated status
# to failed if the results were not processed.
if self.analysis.status != Analysis.STOPPED:
log.warning("Analysis status is '%s' after exit.",
self.analysis.status)
if not config("cuckoo:cuckoo:process_results"):
log.debug("Setting task #%s status to %s",
TASK_FAILED_ANALYSIS)
self.task.write_task_json(status=TASK_FAILED_ANALYSIS)
self.task.set_status(TASK_FAILED_ANALYSIS)
if config("cuckoo:cuckoo:process_results"):
if self.processing_success:
log.debug("Setting task #%s status to %s", self.task.id,
TASK_REPORTED)
self.task.write_task_json(status=TASK_REPORTED)
self.task.set_status(TASK_REPORTED)
else:
log.debug("Setting task #%s status to %s", self.task.id,
TASK_FAILED_PROCESSING)
self.task.write_task_json(status=TASK_FAILED_PROCESSING)
self.task.set_status(TASK_FAILED_PROCESSING)
task_log_stop(self.task.id)
def process_task(task):
db = Database()
try:
task_log_start(task["id"])
logger(
"Starting task reporting",
action="task.report", status="pending",
target=task["target"], category=task["category"],
package=task["package"], options=emit_options(task["options"]),
custom=task["custom"]
)
if task["category"] == "file" and task.get("sample_id"):
sample = db.view_sample(task["sample_id"])
copy_path = cwd("storage", "binaries", sample.sha256)
else:
copy_path = None
try:
process(task["target"], copy_path, task)
db.set_status(task["id"], TASK_REPORTED)
except Exception as e:
log.exception("Task #%d: error reporting: %s", task["id"], e)
db.set_status(task["id"], TASK_FAILED_PROCESSING)
log.info("Task #%d: reports generation completed", task["id"], extra={
"action": "task.report", "status": "success",
})
except Exception as e:
log.exception("Caught unknown exception: %s", e)
finally:
task_log_stop(task["id"])
def handle(self):
ip, port = self.client_address
self.storagepath = self.server.build_storage_path(ip)
if not self.storagepath:
return
task, _ = self.server.get_ctx_for_ip(ip)
task_log_start(task.id)
# Create all missing folders for this analysis.
self.create_folders()
try:
# Initialize the protocol handler class for this connection.
self.negotiate_protocol()
for event in self.protocol:
if isinstance(self.protocol, BsonParser) and event["type"] == "process":
self.open_process_log(event)
except CuckooResultError as e:
log.warning(
"ResultServer connection stopping because of "
"CuckooResultError: %s.", e
)
except (Disconnect, socket.error):
pass
except:
log.exception("FIXME - exception in resultserver connection %s",
self.client_address)
task_log_stop(task.id)
def test_open_process_log_unicode(p):
set_cwd(tempfile.mkdtemp())
cuckoo_create()
mkdir(cwd(analysis=1))
mkdir(cwd("logs", analysis=1))
request = server = mock.MagicMock()
class Handler(ResultHandler):
storagepath = cwd(analysis=1)
def handle(self):
pass
init_logging(logging.DEBUG)
try:
task_log_start(1)
Handler(request, (None, None), server).open_process_log({
"pid":
1,
"ppid":
2,
"process_name":
u"\u202e",
"track":
True,
})
finally:
task_log_stop(1)
def handle(self):
ip, port = self.client_address
self.storagepath = self.server.build_storage_path(ip)
if not self.storagepath:
return
task, _ = self.server.get_ctx_for_ip(ip)
task_log_start(task.id)
# Create all missing folders for this analysis.
self.create_folders()
try:
# Initialize the protocol handler class for this connection.
self.negotiate_protocol()
for event in self.protocol:
if isinstance(self.protocol, BsonParser) and event["type"] == "process":
self.open_process_log(event)
except CuckooResultError as e:
log.warning(
"ResultServer connection stopping because of "
"CuckooResultError: %s.", e
)
except (Disconnect, socket.error):
pass
except:
log.exception("FIXME - exception in resultserver connection %s",
self.client_address)
task_log_stop(task.id)
def handle(self, sock, addr):
"""Handle the incoming connection.
Gevent will close the socket when the function returns."""
ipaddr = addr[0]
with self.task_mgmt_lock:
task_id = self.tasks.get(ipaddr)
if not task_id:
log.warning("ResultServer did not have a task for IP %s",
ipaddr)
return
storagepath = cwd(analysis=task_id)
ctx = HandlerContext(task_id, storagepath, sock)
task_log_start(task_id)
try:
try:
protocol = self.negotiate_protocol(task_id, ctx)
except EOFError:
return
# Registering the context allows us to abort the handler by
# shutting down its socket when the task is deleted; this should
# prevent lingering sockets
with self.task_mgmt_lock:
# NOTE: the task may have been cancelled during the negotation
# protocol and a different task for that IP address may have
# been registered
if self.tasks.get(ipaddr) != task_id:
log.warning(
"Task #%s for IP %s was cancelled during "
"negotiation",
task_id,
ipaddr,
)
return
s = self.handlers.setdefault(task_id, set())
s.add(ctx)
try:
with protocol:
protocol.handle()
except CuckooOperationalError as e:
log.error(e)
finally:
with self.task_mgmt_lock:
s.discard(ctx)
ctx.cancel()
if ctx.buf:
# This is usually not a good sign
log.warning(
"Task #%s with protocol %s has unprocessed "
"data before getting disconnected",
task_id,
protocol,
)
finally:
task_log_stop(task_id)
def test_am_init_duplicate_analysis():
am = am_init()
Folders.create(cwd(analysis=1234))
assert am.init() is False
# Manually disable per-task logging initiated by init().
task_log_stop(1234)
def test_am_init_duplicate_analysis():
am = am_init()
Folders.create(cwd(analysis=1234))
assert am.init() is False
# Manually disable per-task logging initiated by init().
task_log_stop(1234)
def run(self):
"""Run manager thread."""
global active_analysis_count
active_analysis_count += 1
try:
self.launch_analysis()
self.db.set_status(self.task.id, TASK_COMPLETED)
log.debug("Released database task #%d", self.task.id)
if self.cfg.cuckoo.process_results:
# this updates self.task so processing gets the latest and greatest
self.store_task_info()
self.process_results()
self.db.set_status(self.task.id, TASK_REPORTED)
# We make a symbolic link ("latest") which links to the latest
# analysis - this is useful for debugging purposes. This is only
# supported under systems that support symbolic links.
if hasattr(os, "symlink"):
latest = cwd("storage", "analyses", "latest")
# First we have to remove the existing symbolic link, then we
# have to create the new one.
# Deal with race conditions using a lock.
latest_symlink_lock.acquire()
try:
# As per documentation, lexists() returns True for dead
# symbolic links.
if os.path.lexists(latest):
os.remove(latest)
os.symlink(self.storage, latest)
except OSError as e:
log.warning("Error pointing latest analysis symlink: %s" %
e)
finally:
latest_symlink_lock.release()
# overwrite task.json so we have the latest data inside
self.store_task_info()
log.info("Task #%d: analysis procedure completed",
self.task.id,
extra={
"action": "task.stop",
"status": "success",
})
except:
log.exception("Failure in AnalysisManager.run",
extra={
"action": "task.stop",
"status": "error",
})
task_log_stop(self.task.id)
active_analysis_count -= 1
def run(self):
"""Run manager thread."""
global active_analysis_count
active_analysis_count += 1
try:
self.launch_analysis()
self.db.set_status(self.task.id, TASK_COMPLETED)
log.debug("Released database task #%d", self.task.id)
if self.cfg.cuckoo.process_results:
# this updates self.task so processing gets the latest and greatest
self.store_task_info()
self.process_results()
self.db.set_status(self.task.id, TASK_REPORTED)
# We make a symbolic link ("latest") which links to the latest
# analysis - this is useful for debugging purposes. This is only
# supported under systems that support symbolic links.
if hasattr(os, "symlink"):
latest = cwd("storage", "analyses", "latest")
# First we have to remove the existing symbolic link, then we
# have to create the new one.
# Deal with race conditions using a lock.
latest_symlink_lock.acquire()
try:
# As per documentation, lexists() returns True for dead
# symbolic links.
if os.path.lexists(latest):
os.remove(latest)
os.symlink(self.storage, latest)
except OSError as e:
log.warning("Error pointing latest analysis symlink: %s" % e)
finally:
latest_symlink_lock.release()
# overwrite task.json so we have the latest data inside
self.store_task_info()
log.info(
"Task #%d: analysis procedure completed",
self.task.id, extra={
"action": "task.stop",
"status": "success",
}
)
except:
log.exception("Failure in AnalysisManager.run", extra={
"action": "task.stop",
"status": "error",
})
task_log_stop(self.task.id)
active_analysis_count -= 1
def test_am_init_success():
am = am_init()
assert am.init() is True
assert os.path.exists(cwd(analysis=1234))
assert os.path.exists(cwd("storage", "binaries", sha256_))
assert os.path.exists(cwd("binary", analysis=1234))
# Manually disable per-task logging initiated by init().
task_log_stop(1234)
def test_am_init_success():
am = am_init()
assert am.init() is True
assert os.path.exists(cwd(analysis=1234))
assert os.path.exists(cwd("storage", "binaries", sha256_))
assert os.path.exists(cwd("binary", analysis=1234))
# Manually disable per-task logging initiated by init().
task_log_stop(1234)
def test_dump_memory_unicode(self):
p1 = mock.MagicMock()
p1.communicate.return_value = "5.0.28r111378", ""
p1.returncode = 0
p2 = mock.MagicMock()
p2.wait.return_value = None
mkdir(cwd(analysis=1))
task_log_start(1)
init_logging(logging.DEBUG)
with mock.patch("cuckoo.machinery.virtualbox.Popen") as p:
p.side_effect = p1, p2
self.m.dump_memory("label", u"mem\u202eory.dmp")
task_log_stop(1)
def test_dump_memory_unicode(self):
p1 = mock.MagicMock()
p1.communicate.return_value = "5.0.28r111378", ""
p1.returncode = 0
p2 = mock.MagicMock()
p2.wait.return_value = None
mkdir(cwd(analysis=1))
task_log_start(1)
init_logging(logging.DEBUG)
with mock.patch("cuckoo.machinery.virtualbox.Popen") as p:
p.side_effect = p1, p2
self.m.dump_memory("label", u"mem\u202eory.dmp")
task_log_stop(1)
def process_task(task):
db = Database()
if not task.dir_exists():
log.error("Task #%s directory %s does not exist, cannot process it",
task.id, task.path)
db.set_status(task.id, TASK_FAILED_PROCESSING)
return
task_log_start(task.id)
if task.targets:
target = task.targets[0]
else:
target = Target()
logger("Starting task reporting",
action="task.report",
status="pending",
target=target.target,
category=target.category,
package=task["package"],
options=emit_options(task["options"]),
custom=task["custom"])
success = False
try:
success = task.process()
except Exception as e:
log.error("Failed to process task #%s. Error: %s", task.id, e)
finally:
if success:
log.info("Task #%d: reports generation completed",
task.id,
extra={
"action": "task.report",
"status": "success",
})
db.set_status(task.id, TASK_REPORTED)
else:
log.error("Failed to process task #%s",
task.id,
extra={
"action": "task.report",
"status": "failed",
})
db.set_status(task.id, TASK_FAILED_PROCESSING)
task_log_stop(task.id)
def test_open_process_log_unicode(p):
set_cwd(tempfile.mkdtemp())
cuckoo_create()
mkdir(cwd(analysis=1))
mkdir(cwd("logs", analysis=1))
request = server = mock.MagicMock()
class Handler(ResultHandler):
storagepath = cwd(analysis=1)
def handle(self):
pass
init_logging(logging.DEBUG)
try:
task_log_start(1)
Handler(request, (None, None), server).open_process_log({
"pid": 1, "ppid": 2, "process_name": u"\u202e", "track": True,
})
finally:
task_log_stop(1)
def finalize(self, db):
self.ev_client.stop()
self.task.set_latest()
if self.machine.locked:
log.debug("Releasing machine lock on %s", self.machine.label)
self.machine = self.machinery.release(self.machine.label)
self.release_machine_lock()
with open(cwd("pids_targets.json", analysis=self.task.id), "wb") as fp:
json.dump(self.all_pids_targets, fp, indent=2)
if self.analysis.status != Analysis.STOPPED:
log.warning("Analysis status is '%s' after exit.",
self.analysis.status)
self.task.write_task_json(status=TASK_FAILED_ANALYSIS)
self.task.set_status(TASK_FAILED_ANALYSIS)
if self.completed:
log.info("Setting task #%d to reported", self.task.id)
self.task.write_task_json(status=TASK_REPORTED)
self.task.set_status(TASK_REPORTED)
task_log_stop(self.task.id)
