def set_result(file_ext_id, probe, result): with session_transaction() as session: file_ext = FileExt.load_from_ext_id(file_ext_id, session=session) sanitized_res = _sanitize_res(result) file_ext.set_result(probe, sanitized_res) scan_id = file_ext.scan.external_id log.info("scan %s: file %s result from %s", scan_id, file_ext_id, probe) is_finished(scan_id)
def set_result(file_ext_id, probe, result): with session_transaction() as session: file_ext = FileExt.load_from_ext_id(file_ext_id, session=session) sanitized_res = _sanitize_res(result) file_ext.set_result(probe, sanitized_res) scan_id = file_ext.scan.external_id log.info("scan %s: file %s result from %s", scan_id, file_ext_id, probe) is_finished(scan_id)
def handle_output_files(file_ext_id, result, error_case=False): log.info("Handling output for file %s", file_ext_id) with session_transaction() as session: file_ext = FileExt.load_from_ext_id(file_ext_id, session) scan = file_ext.scan uploaded_files = result.get('uploaded_files', None) log.debug("scan %s file %s depth %s", scan.external_id, file_ext_id, file_ext.depth) if uploaded_files is None: return resubmit = scan.resubmit_files max_resubmit_level = get_max_resubmit_level() if max_resubmit_level != 0 and file_ext.depth > \ max_resubmit_level: log.warning("scan %s file %s resubmit level %s exceeded max " "level (%s)", scan.external_id, file_ext_id, file_ext.depth, max_resubmit_level ) resubmit = False if not resubmit or error_case: reason = "Error case" if error_case else "Resubmit disabled" log.debug("scan %s: %s flushing files", scan.external_id, reason) celery_brain.files_flush(list(uploaded_files.values()), scan.external_id) return log.debug("scan %s: found files %s", scan.external_id, uploaded_files) # Retrieve the DB probe_result to link it with # a new FileProbeResult in _append_new_files probe_result = file_ext.fetch_probe_result(result['name']) new_fws = _append_new_files_to_scan(scan, uploaded_files, probe_result, file_ext.depth+1) parent_file = file_ext.file for new_fw in new_fws: parent_file.children.append(new_fw) session.commit() log.debug("scan %s: %d new files to resubmit", scan.external_id, len(new_fws)) scan_request = _create_scan_request(new_fws, scan.get_probelist(), scan.mimetype_filtering) scan_request = _add_empty_results(new_fws, scan_request, scan, session) if scan_request.nb_files == 0: scan.set_status(IrmaScanStatus.finished) log.info("scan %s: nothing to do flushing files", scan.external_id) celery_brain.files_flush(list(uploaded_files.values()), scan.external_id) return for new_fw in new_fws: celery_brain.scan_launch(new_fw.external_id, new_fw.probes, scan.external_id) return
def handle_output_files(file_ext_id, result, error_case=False): log.info("Handling output for file %s", file_ext_id) with session_transaction() as session: file_ext = FileExt.load_from_ext_id(file_ext_id, session) scan = file_ext.scan uploaded_files = result.get('uploaded_files', None) log.debug("scan %s file %s depth %s", scan.external_id, file_ext_id, file_ext.depth) if uploaded_files is None: return resubmit = scan.resubmit_files max_resubmit_level = get_max_resubmit_level() if max_resubmit_level != 0 and file_ext.depth > \ max_resubmit_level: log.warning( "scan %s file %s resubmit level %s exceeded max " "level (%s)", scan.external_id, file_ext_id, file_ext.depth, max_resubmit_level) resubmit = False if not resubmit or error_case: reason = "Error case" if error_case else "Resubmit disabled" log.debug("scan %s: %s flushing files", scan.external_id, reason) celery_brain.files_flush(list(uploaded_files.values()), scan.external_id) return log.debug("scan %s: found files %s", scan.external_id, uploaded_files) # Retrieve the DB probe_result to link it with # a new FileProbeResult in _append_new_files probe_result = file_ext.fetch_probe_result(result['name']) new_fws = _append_new_files_to_scan(scan, uploaded_files, probe_result, file_ext.depth + 1) parent_file = file_ext.file for new_fw in new_fws: parent_file.children.append(new_fw) session.commit() log.debug("scan %s: %d new files to resubmit", scan.external_id, len(new_fws)) scan_request = _create_scan_request(new_fws, scan.get_probelist(), scan.mimetype_filtering) scan_request = _add_empty_results(new_fws, scan_request, scan, session) if scan_request.nb_files == 0: scan.set_status(IrmaScanStatus.finished) log.info("scan %s: nothing to do flushing files", scan.external_id) celery_brain.files_flush(list(uploaded_files.values()), scan.external_id) return for new_fw in new_fws: celery_brain.scan_launch(new_fw.external_id, new_fw.probes, scan.external_id) return
def scan_launch_file_ext(file_ext_id): file_ext = None with session_transaction() as session: try: file_ext = FileExt.load_from_ext_id(file_ext_id, session) scan_id = file_ext.scan.external_id log.debug("scan %s: launch scan for file_ext: %s", scan_id, file_ext_id) ftp_ctrl.upload_file(file_ext_id, file_ext.file.path) # launch new celery scan task on brain celery_brain.scan_launch(file_ext_id, file_ext.probes, scan_id) except IrmaFtpError as e: log.error("file_ext %s: ftp upload error %s", file_ext_id, str(e)) if file_ext is not None: file_ext.scan.set_status(IrmaScanStatus.error_ftp_upload) except Exception as e: log.exception(e)
def quick_scan(request): """ Launch a scan for one file """ session = db.session ip = request.remote_addr # Create file fe = files_ctrl.create(request) fe = FileExt.load_from_ext_id(fe['result_id'], session) # Create a scan scan = Scan(compat.timestamp(), ip, files_ext=[fe, ]) session.add(scan) session.commit() # launch_asynchronous scan via frontend task celery_frontend.scan_launch(str(scan.external_id)) return scan_schema.dump(scan).data
def quick_scan(request): """ Launch a scan for one file """ session = db.session ip = request.remote_addr # Create file fe = files_ctrl.create(request) fe = FileExt.load_from_ext_id(fe['result_id'], session) # Create a scan scan = Scan(compat.timestamp(), ip, files_ext=[fe, ]) session.add(scan) session.commit() # launch_asynchronous scan via frontend task celery_frontend.scan_launch(str(scan.external_id)) return scan_schema.dump(scan).data
def launch_v2(request, body): """ Launch a scan. The request should be performed using a POST request method. The input json format is the following: { files: [fileext1, fileext2...] options: probes: list of probes or None for all available, force: boolean (default False), mimetype_filtering: boolean (default True), resubmit_files: boolean (default True), } """ scan_params = body if not scan_params: raise HTTPInvalidParam("Missing json parameters", "body") files_list = body.get('files', None) if files_list is None or len(files_list) == 0: raise HTTPInvalidParam("Missing values", "files") # Set default values force = True mimetype_filtering = True resubmit_files = True probes = None # override with given values if set scan_options = body.get("options", None) if scan_options is not None: force = scan_options.get("force", force) if type(force) is not bool: raise HTTPInvalidParam("Should be boolean", "force") mimetype_filtering = scan_options.get("mimetype_filtering", mimetype_filtering) if type(mimetype_filtering) is not bool: raise HTTPInvalidParam("Should be boolean", "mimetype_filtering") resubmit_files = scan_options.get("resubmit_files", resubmit_files) if type(resubmit_files) is not bool: raise HTTPInvalidParam("Should be boolean", "resubmit_files") probes = scan_options.get("probes", probes) session = db.session ip = request.remote_addr files_ext = [] for fe_id in files_list: try: file_ext = FileExt.load_from_ext_id(fe_id, session) except IrmaDatabaseResultNotFound: raise HTTPInvalidParam("File %s not found" % fe_id, "files") if file_ext.file.path is None: raise HTTPInvalidParam("File with hash %s should be (" "re)uploaded" % file_ext.file.sha256, "files") if file_ext.scan is not None: raise HTTPInvalidParam("File %s already scanned" % fe_id, "files") files_ext.append(file_ext) scan = Scan(compat.timestamp(), ip, force=force, mimetype_filtering=mimetype_filtering, resubmit_files=resubmit_files, files_ext=files_ext, probes=probes) session.add(scan) session.commit() # launch_asynchronous scan via frontend task celery_frontend.scan_launch(str(scan.external_id)) return scan_schema.dump(scan).data
def launch_v2(request, body): """ Launch a scan. The request should be performed using a POST request method. The input json format is the following: { files: [fileext1, fileext2...] options: probes: list of probes or None for all available, force: boolean (default False), mimetype_filtering: boolean (default True), resubmit_files: boolean (default True), } """ scan_params = body if not scan_params: raise HTTPInvalidParam("Missing json parameters", "body") files_list = body.get('files', None) if files_list is None or len(files_list) == 0: raise HTTPInvalidParam("Missing values", "files") # Set default values force = True mimetype_filtering = True resubmit_files = True probes = None # override with given values if set scan_options = body.get("options", None) if scan_options is not None: force = scan_options.get("force", force) if type(force) is not bool: raise HTTPInvalidParam("Should be boolean", "force") mimetype_filtering = scan_options.get("mimetype_filtering", mimetype_filtering) if type(mimetype_filtering) is not bool: raise HTTPInvalidParam("Should be boolean", "mimetype_filtering") resubmit_files = scan_options.get("resubmit_files", resubmit_files) if type(resubmit_files) is not bool: raise HTTPInvalidParam("Should be boolean", "resubmit_files") probes = scan_options.get("probes", probes) session = db.session ip = request.remote_addr files_ext = [] for fe_id in files_list: try: file_ext = FileExt.load_from_ext_id(fe_id, session) except IrmaDatabaseResultNotFound: raise HTTPInvalidParam("File %s not found" % fe_id, "files") if file_ext.file.path is None: raise HTTPInvalidParam("File with hash %s should be (" "re)uploaded" % file_ext.file.sha256, "files") if file_ext.scan is not None: raise HTTPInvalidParam("File %s already scanned" % fe_id, "files") files_ext.append(file_ext) scan = Scan(compat.timestamp(), ip, force=force, mimetype_filtering=mimetype_filtering, resubmit_files=resubmit_files, files_ext=files_ext, probes=probes) session.add(scan) session.commit() # launch_asynchronous scan via frontend task celery_frontend.scan_launch(str(scan.external_id)) return scan_schema.dump(scan).data
def test_load_from_ext_id_raises_noresult(self): m_session = MagicMock() ext_id = "whatever" m_session.query().filter().one.side_effect = NoResultFound() with self.assertRaises(IrmaDatabaseResultNotFound): FileExt.load_from_ext_id(ext_id, m_session)
def test_load_from_ext_id_raises_multiple(self): m_session = MagicMock() ext_id = "whatever" m_session.query().filter().one.side_effect = MultipleResultsFound() with self.assertRaises(IrmaDatabaseError): FileExt.load_from_ext_id(ext_id, m_session)
def test_load_from_ext_id(self): m_session = MagicMock() ext_id = "whatever" FileExt.load_from_ext_id(ext_id, m_session) m_filter = m_session.query(FileExt).filter m_filter.is_called_once_with(FileExt.external_id == ext_id)
def launch_v2(request, body): """ Launch a scan. The request should be performed using a POST request method. The input json format is the following: { files: [fileext1, fileext2...] options: probes: list of probes or None for all available, force: boolean (default False), mimetype_filtering: boolean (default True), resubmit_files: boolean (default True), } """ scan_params = body if not scan_params: raise HTTPInvalidParam("Missing json parameters", "body") files_list = body.get('files', None) if files_list is None or len(files_list) == 0: raise HTTPInvalidParam("Missing values", "files") # Set default values force = True mimetype_filtering = True resubmit_files = True probes = None # override with given values if set scan_options = body.get("options", None) if scan_options is not None: force = scan_options.get("force", False) if type(force) is not bool: raise HTTPInvalidParam("Should be boolean", "force") mimetype_filtering = scan_options.get("mimetype_filtering", True) if type(mimetype_filtering) is not bool: raise HTTPInvalidParam("Should be boolean", "mimetype_filtering") resubmit_files = scan_options.get("resubmit_files", True) if type(resubmit_files) is not bool: raise HTTPInvalidParam("Should be boolean", "resubmit_files") probes = scan_options.get("probes", None) session = db.session ip = request.remote_addr scan = Scan(compat.timestamp(), ip) session.add(scan) # handle scan parameter # cached results: "force" (default: True) scan.force = force # use mimetype for probelist: "mimetype_filtering" (default: True) scan.mimetype_filtering = mimetype_filtering # rescan file outputted from probes "resubmit_files" (default: True) scan.resubmit_files = resubmit_files scan.set_status(IrmaScanStatus.empty) session.commit() log.debug("scan %s: created", scan.external_id) msg = "scan %s: Force %s MimeF %s" msg += " Resub %s Probes %s" log.debug(msg, scan.external_id, scan.force, scan.mimetype_filtering, scan.resubmit_files, probes) for fe_id in files_list: log.info("scan %s adding file %s", scan.external_id, fe_id) try: file_ext = FileExt.load_from_ext_id(fe_id, session) except IrmaDatabaseResultNotFound: raise HTTPInvalidParam("File %s not found" % fe_id, "files") if file_ext.file.path is None: raise HTTPInvalidParam( "File with hash %s should be (" "re)uploaded" % file_ext.file.sha256, "files") if file_ext.scan is not None: raise HTTPInvalidParam("File %s already scanned" % fe_id, "files") file_ext.scan = scan scan.set_status(IrmaScanStatus.ready) session.commit() probelist = probe_ctrl.check_probe(probes) scan.set_probelist(probelist) session.commit() # launch_asynchronous scan via frontend task celery_frontend.scan_launch(str(scan.external_id)) return scan_schema.dump(scan).data
def test_load_from_ext_id_raises_multiple(self): m_session = MagicMock() ext_id = "whatever" m_session.query().filter().one.side_effect = MultipleResultsFound() with self.assertRaises(IrmaDatabaseError): FileExt.load_from_ext_id(ext_id, m_session)
def test_load_from_ext_id_raises_noresult(self): m_session = MagicMock() ext_id = "whatever" m_session.query().filter().one.side_effect = NoResultFound() with self.assertRaises(IrmaDatabaseResultNotFound): FileExt.load_from_ext_id(ext_id, m_session)
def test_load_from_ext_id(self): m_session = MagicMock() ext_id = "whatever" FileExt.load_from_ext_id(ext_id, m_session) m_filter = m_session.query(FileExt).filter m_filter.is_called_once_with(FileExt.external_id == ext_id)