def set_result(file_ext_id, probe, result):
with session_transaction() as session:
file_ext = FileExt.load_from_ext_id(file_ext_id, session=session)
sanitized_res = _sanitize_res(result)
file_ext.set_result(probe, sanitized_res)
scan_id = file_ext.scan.external_id
log.info("scan %s: file %s result from %s", scan_id, file_ext_id,
probe)
is_finished(scan_id)
def set_result(file_ext_id, probe, result):
with session_transaction() as session:
file_ext = FileExt.load_from_ext_id(file_ext_id, session=session)
sanitized_res = _sanitize_res(result)
file_ext.set_result(probe, sanitized_res)
scan_id = file_ext.scan.external_id
log.info("scan %s: file %s result from %s",
scan_id, file_ext_id, probe)
is_finished(scan_id)
def handle_output_files(file_ext_id, result, error_case=False):
log.info("Handling output for file %s", file_ext_id)
with session_transaction() as session:
file_ext = FileExt.load_from_ext_id(file_ext_id, session)
scan = file_ext.scan
uploaded_files = result.get('uploaded_files', None)
log.debug("scan %s file %s depth %s", scan.external_id,
file_ext_id, file_ext.depth)
if uploaded_files is None:
return
resubmit = scan.resubmit_files
max_resubmit_level = get_max_resubmit_level()
if max_resubmit_level != 0 and file_ext.depth > \
max_resubmit_level:
log.warning("scan %s file %s resubmit level %s exceeded max "
"level (%s)", scan.external_id,
file_ext_id, file_ext.depth,
max_resubmit_level
)
resubmit = False
if not resubmit or error_case:
reason = "Error case" if error_case else "Resubmit disabled"
log.debug("scan %s: %s flushing files", scan.external_id, reason)
celery_brain.files_flush(list(uploaded_files.values()),
scan.external_id)
return
log.debug("scan %s: found files %s", scan.external_id, uploaded_files)
# Retrieve the DB probe_result to link it with
# a new FileProbeResult in _append_new_files
probe_result = file_ext.fetch_probe_result(result['name'])
new_fws = _append_new_files_to_scan(scan, uploaded_files,
probe_result, file_ext.depth+1)
parent_file = file_ext.file
for new_fw in new_fws:
parent_file.children.append(new_fw)
session.commit()
log.debug("scan %s: %d new files to resubmit",
scan.external_id, len(new_fws))
scan_request = _create_scan_request(new_fws,
scan.get_probelist(),
scan.mimetype_filtering)
scan_request = _add_empty_results(new_fws, scan_request, scan, session)
if scan_request.nb_files == 0:
scan.set_status(IrmaScanStatus.finished)
log.info("scan %s: nothing to do flushing files",
scan.external_id)
celery_brain.files_flush(list(uploaded_files.values()),
scan.external_id)
return
for new_fw in new_fws:
celery_brain.scan_launch(new_fw.external_id,
new_fw.probes,
scan.external_id)
return
def handle_output_files(file_ext_id, result, error_case=False):
log.info("Handling output for file %s", file_ext_id)
with session_transaction() as session:
file_ext = FileExt.load_from_ext_id(file_ext_id, session)
scan = file_ext.scan
uploaded_files = result.get('uploaded_files', None)
log.debug("scan %s file %s depth %s", scan.external_id, file_ext_id,
file_ext.depth)
if uploaded_files is None:
return
resubmit = scan.resubmit_files
max_resubmit_level = get_max_resubmit_level()
if max_resubmit_level != 0 and file_ext.depth > \
max_resubmit_level:
log.warning(
"scan %s file %s resubmit level %s exceeded max "
"level (%s)", scan.external_id, file_ext_id, file_ext.depth,
max_resubmit_level)
resubmit = False
if not resubmit or error_case:
reason = "Error case" if error_case else "Resubmit disabled"
log.debug("scan %s: %s flushing files", scan.external_id, reason)
celery_brain.files_flush(list(uploaded_files.values()),
scan.external_id)
return
log.debug("scan %s: found files %s", scan.external_id, uploaded_files)
# Retrieve the DB probe_result to link it with
# a new FileProbeResult in _append_new_files
probe_result = file_ext.fetch_probe_result(result['name'])
new_fws = _append_new_files_to_scan(scan, uploaded_files, probe_result,
file_ext.depth + 1)
parent_file = file_ext.file
for new_fw in new_fws:
parent_file.children.append(new_fw)
session.commit()
log.debug("scan %s: %d new files to resubmit", scan.external_id,
len(new_fws))
scan_request = _create_scan_request(new_fws, scan.get_probelist(),
scan.mimetype_filtering)
scan_request = _add_empty_results(new_fws, scan_request, scan, session)
if scan_request.nb_files == 0:
scan.set_status(IrmaScanStatus.finished)
log.info("scan %s: nothing to do flushing files", scan.external_id)
celery_brain.files_flush(list(uploaded_files.values()),
scan.external_id)
return
for new_fw in new_fws:
celery_brain.scan_launch(new_fw.external_id, new_fw.probes,
scan.external_id)
return
def scan_launch_file_ext(file_ext_id):
file_ext = None
with session_transaction() as session:
try:
file_ext = FileExt.load_from_ext_id(file_ext_id, session)
scan_id = file_ext.scan.external_id
log.debug("scan %s: launch scan for file_ext: %s",
scan_id, file_ext_id)
ftp_ctrl.upload_file(file_ext_id, file_ext.file.path)
# launch new celery scan task on brain
celery_brain.scan_launch(file_ext_id, file_ext.probes, scan_id)
except IrmaFtpError as e:
log.error("file_ext %s: ftp upload error %s", file_ext_id, str(e))
if file_ext is not None:
file_ext.scan.set_status(IrmaScanStatus.error_ftp_upload)
except Exception as e:
log.exception(e)
def quick_scan(request):
""" Launch a scan for one file
"""
session = db.session
ip = request.remote_addr
# Create file
fe = files_ctrl.create(request)
fe = FileExt.load_from_ext_id(fe['result_id'], session)
# Create a scan
scan = Scan(compat.timestamp(), ip, files_ext=[fe, ])
session.add(scan)
session.commit()
# launch_asynchronous scan via frontend task
celery_frontend.scan_launch(str(scan.external_id))
return scan_schema.dump(scan).data
def quick_scan(request):
""" Launch a scan for one file
"""
session = db.session
ip = request.remote_addr
# Create file
fe = files_ctrl.create(request)
fe = FileExt.load_from_ext_id(fe['result_id'], session)
# Create a scan
scan = Scan(compat.timestamp(), ip, files_ext=[fe, ])
session.add(scan)
session.commit()
# launch_asynchronous scan via frontend task
celery_frontend.scan_launch(str(scan.external_id))
return scan_schema.dump(scan).data
def launch_v2(request, body):
""" Launch a scan.
The request should be performed using a POST request method.
The input json format is the following:
{
files: [fileext1, fileext2...]
options:
probes: list of probes or None for all available,
force: boolean (default False),
mimetype_filtering: boolean (default True),
resubmit_files: boolean (default True),
}
"""
scan_params = body
if not scan_params:
raise HTTPInvalidParam("Missing json parameters", "body")
files_list = body.get('files', None)
if files_list is None or len(files_list) == 0:
raise HTTPInvalidParam("Missing values", "files")
# Set default values
force = True
mimetype_filtering = True
resubmit_files = True
probes = None
# override with given values if set
scan_options = body.get("options", None)
if scan_options is not None:
force = scan_options.get("force", force)
if type(force) is not bool:
raise HTTPInvalidParam("Should be boolean", "force")
mimetype_filtering = scan_options.get("mimetype_filtering",
mimetype_filtering)
if type(mimetype_filtering) is not bool:
raise HTTPInvalidParam("Should be boolean",
"mimetype_filtering")
resubmit_files = scan_options.get("resubmit_files", resubmit_files)
if type(resubmit_files) is not bool:
raise HTTPInvalidParam("Should be boolean",
"resubmit_files")
probes = scan_options.get("probes", probes)
session = db.session
ip = request.remote_addr
files_ext = []
for fe_id in files_list:
try:
file_ext = FileExt.load_from_ext_id(fe_id, session)
except IrmaDatabaseResultNotFound:
raise HTTPInvalidParam("File %s not found" % fe_id,
"files")
if file_ext.file.path is None:
raise HTTPInvalidParam("File with hash %s should be ("
"re)uploaded" %
file_ext.file.sha256,
"files")
if file_ext.scan is not None:
raise HTTPInvalidParam("File %s already scanned" %
fe_id,
"files")
files_ext.append(file_ext)
scan = Scan(compat.timestamp(), ip,
force=force, mimetype_filtering=mimetype_filtering,
resubmit_files=resubmit_files, files_ext=files_ext,
probes=probes)
session.add(scan)
session.commit()
# launch_asynchronous scan via frontend task
celery_frontend.scan_launch(str(scan.external_id))
return scan_schema.dump(scan).data
def launch_v2(request, body):
""" Launch a scan.
The request should be performed using a POST request method.
The input json format is the following:
{
files: [fileext1, fileext2...]
options:
probes: list of probes or None for all available,
force: boolean (default False),
mimetype_filtering: boolean (default True),
resubmit_files: boolean (default True),
}
"""
scan_params = body
if not scan_params:
raise HTTPInvalidParam("Missing json parameters", "body")
files_list = body.get('files', None)
if files_list is None or len(files_list) == 0:
raise HTTPInvalidParam("Missing values", "files")
# Set default values
force = True
mimetype_filtering = True
resubmit_files = True
probes = None
# override with given values if set
scan_options = body.get("options", None)
if scan_options is not None:
force = scan_options.get("force", force)
if type(force) is not bool:
raise HTTPInvalidParam("Should be boolean", "force")
mimetype_filtering = scan_options.get("mimetype_filtering",
mimetype_filtering)
if type(mimetype_filtering) is not bool:
raise HTTPInvalidParam("Should be boolean",
"mimetype_filtering")
resubmit_files = scan_options.get("resubmit_files", resubmit_files)
if type(resubmit_files) is not bool:
raise HTTPInvalidParam("Should be boolean",
"resubmit_files")
probes = scan_options.get("probes", probes)
session = db.session
ip = request.remote_addr
files_ext = []
for fe_id in files_list:
try:
file_ext = FileExt.load_from_ext_id(fe_id, session)
except IrmaDatabaseResultNotFound:
raise HTTPInvalidParam("File %s not found" % fe_id,
"files")
if file_ext.file.path is None:
raise HTTPInvalidParam("File with hash %s should be ("
"re)uploaded" %
file_ext.file.sha256,
"files")
if file_ext.scan is not None:
raise HTTPInvalidParam("File %s already scanned" %
fe_id,
"files")
files_ext.append(file_ext)
scan = Scan(compat.timestamp(), ip,
force=force, mimetype_filtering=mimetype_filtering,
resubmit_files=resubmit_files, files_ext=files_ext,
probes=probes)
session.add(scan)
session.commit()
# launch_asynchronous scan via frontend task
celery_frontend.scan_launch(str(scan.external_id))
return scan_schema.dump(scan).data
def test_load_from_ext_id_raises_noresult(self):
m_session = MagicMock()
ext_id = "whatever"
m_session.query().filter().one.side_effect = NoResultFound()
with self.assertRaises(IrmaDatabaseResultNotFound):
FileExt.load_from_ext_id(ext_id, m_session)
def test_load_from_ext_id_raises_multiple(self):
m_session = MagicMock()
ext_id = "whatever"
m_session.query().filter().one.side_effect = MultipleResultsFound()
with self.assertRaises(IrmaDatabaseError):
FileExt.load_from_ext_id(ext_id, m_session)
def test_load_from_ext_id(self):
m_session = MagicMock()
ext_id = "whatever"
FileExt.load_from_ext_id(ext_id, m_session)
m_filter = m_session.query(FileExt).filter
m_filter.is_called_once_with(FileExt.external_id == ext_id)
def launch_v2(request, body):
""" Launch a scan.
The request should be performed using a POST request method.
The input json format is the following:
{
files: [fileext1, fileext2...]
options:
probes: list of probes or None for all available,
force: boolean (default False),
mimetype_filtering: boolean (default True),
resubmit_files: boolean (default True),
}
"""
scan_params = body
if not scan_params:
raise HTTPInvalidParam("Missing json parameters", "body")
files_list = body.get('files', None)
if files_list is None or len(files_list) == 0:
raise HTTPInvalidParam("Missing values", "files")
# Set default values
force = True
mimetype_filtering = True
resubmit_files = True
probes = None
# override with given values if set
scan_options = body.get("options", None)
if scan_options is not None:
force = scan_options.get("force", False)
if type(force) is not bool:
raise HTTPInvalidParam("Should be boolean", "force")
mimetype_filtering = scan_options.get("mimetype_filtering", True)
if type(mimetype_filtering) is not bool:
raise HTTPInvalidParam("Should be boolean", "mimetype_filtering")
resubmit_files = scan_options.get("resubmit_files", True)
if type(resubmit_files) is not bool:
raise HTTPInvalidParam("Should be boolean", "resubmit_files")
probes = scan_options.get("probes", None)
session = db.session
ip = request.remote_addr
scan = Scan(compat.timestamp(), ip)
session.add(scan)
# handle scan parameter
# cached results: "force" (default: True)
scan.force = force
# use mimetype for probelist: "mimetype_filtering" (default: True)
scan.mimetype_filtering = mimetype_filtering
# rescan file outputted from probes "resubmit_files" (default: True)
scan.resubmit_files = resubmit_files
scan.set_status(IrmaScanStatus.empty)
session.commit()
log.debug("scan %s: created", scan.external_id)
msg = "scan %s: Force %s MimeF %s"
msg += " Resub %s Probes %s"
log.debug(msg, scan.external_id, scan.force, scan.mimetype_filtering,
scan.resubmit_files, probes)
for fe_id in files_list:
log.info("scan %s adding file %s", scan.external_id, fe_id)
try:
file_ext = FileExt.load_from_ext_id(fe_id, session)
except IrmaDatabaseResultNotFound:
raise HTTPInvalidParam("File %s not found" % fe_id, "files")
if file_ext.file.path is None:
raise HTTPInvalidParam(
"File with hash %s should be ("
"re)uploaded" % file_ext.file.sha256, "files")
if file_ext.scan is not None:
raise HTTPInvalidParam("File %s already scanned" % fe_id, "files")
file_ext.scan = scan
scan.set_status(IrmaScanStatus.ready)
session.commit()
probelist = probe_ctrl.check_probe(probes)
scan.set_probelist(probelist)
session.commit()
# launch_asynchronous scan via frontend task
celery_frontend.scan_launch(str(scan.external_id))
return scan_schema.dump(scan).data
def test_load_from_ext_id_raises_multiple(self):
m_session = MagicMock()
ext_id = "whatever"
m_session.query().filter().one.side_effect = MultipleResultsFound()
with self.assertRaises(IrmaDatabaseError):
FileExt.load_from_ext_id(ext_id, m_session)
def test_load_from_ext_id_raises_noresult(self):
m_session = MagicMock()
ext_id = "whatever"
m_session.query().filter().one.side_effect = NoResultFound()
with self.assertRaises(IrmaDatabaseResultNotFound):
FileExt.load_from_ext_id(ext_id, m_session)
def test_load_from_ext_id(self):
m_session = MagicMock()
ext_id = "whatever"
FileExt.load_from_ext_id(ext_id, m_session)
m_filter = m_session.query(FileExt).filter
m_filter.is_called_once_with(FileExt.external_id == ext_id)