def mimetype_filter_scan_request(scan_request_dict): try: with session_transaction() as session: scan_request = IrmaScanRequest(scan_request_dict) for file in scan_request.files(): filtered_probelist = [] mimetype = scan_request.get_mimetype(file) mimetype_probelist = probe_ctrl.mimetype_probelist(mimetype, session) probe_list = scan_request.get_probelist(file) log.debug("file: %s probe_list: %s", file, "-".join(probe_list)) # first check probe_list for unknown probe for probe in probe_list: # check if probe exists if probe not in probe_ctrl.available_probes: log.warning("probe %s not available", probe) if probe in mimetype_probelist: # probe is asked and supported by mimetype log.debug("file %s probe %s asked" + " and supported for %s append to request", file, probe, mimetype) filtered_probelist.append(probe) # update probe list in scan request scan_request.set_probelist(file, filtered_probelist) return IrmaTaskReturn.success(scan_request.to_dict()) except Exception as e: log.exception(e) return IrmaTaskReturn.error("Brain error")
def scan(frontend_scanid, scan_request_dict): try: with session_transaction() as session: log.debug("scanid: %s received %s", frontend_scanid, scan_request_dict) user = User.get_by_rmqvhost(session) scan_request = IrmaScanRequest(scan_request_dict) scan = scan_ctrl.new(frontend_scanid, user, scan_request.nb_files, session) available_probelist = probe_ctrl.get_list(session) # Now, create one subtask per file to # scan per probe new_jobs = [] for filehash in scan_request.filehashes(): probelist = scan_request.get_probelist(filehash) scan_ctrl.check_probelist(scan, probelist, available_probelist, session) for probe in probelist: j = Job(scan.id, filehash, probe) session.add(j) new_jobs.append(j) session.commit() scan_ctrl.launch(scan, new_jobs, session) celery_frontend.scan_launched(scan.scan_id, scan_request.to_dict()) log.info("scanid %s: %d file(s) received / %d active probe(s) / " "%d job(s) launched", scan.scan_id, scan.nb_files, len(available_probelist), len(scan.jobs)) except Exception as e: log.exception(e) raise
def scan(file, probelist, frontend_scan): try: with session_transaction() as session: log.debug("scan_id: %s fileweb_id: %s received %s", frontend_scan, file, probelist) user = User.get_by_rmqvhost(session) scan = scan_ctrl.new(frontend_scan, user, session) # Now, create one subtask per file to scan per probe new_jobs = [] for probe in probelist: if probe in probe_ctrl.available_probes: j = Job(scan.id, file, probe) session.add(j) new_jobs.append(j) else: # send an error message to not stuck the scan # One probe asked is no more present res = probe_ctrl.create_error_results(probe, "missing probe", session) celery_frontend.scan_result(file, probe, res) session.commit() scan_ctrl.launch(scan, new_jobs, session) log.info("scan_id %s: file %s received / %d active probe(s) / " "%d job(s) launched", scan.scan_id, file, len(probe_ctrl.available_probes), len(scan.jobs)) except Exception as e: log.exception(e) raise
def mimetype_filter_scan_request(scan_request_dict): try: with session_transaction() as session: available_probelist = probe_ctrl.get_list(session) scan_request = IrmaScanRequest(scan_request_dict) for filehash in scan_request.filehashes(): filtered_probelist = [] mimetype = scan_request.get_mimetype(filehash) mimetype_probelist = probe_ctrl.mimetype_probelist(mimetype, session) probe_list = scan_request.get_probelist(filehash) log.debug("filehash: %s probe_list: %s", filehash, "-".join(probe_list)) # first check probe_list for unknown probe for probe in probe_list: # check if probe exists if probe not in available_probelist: msg = "Unknown probe {0}".format(probe) log.error(msg) return IrmaTaskReturn.error(msg) if probe in mimetype_probelist: # probe is asked but not supported by mimetype log.debug("filehash %s probe %s asked but" + " not supported for %s", filehash, probe, mimetype) filtered_probelist.append(probe) # update probe list in scan request scan_request.set_probelist(filehash, filtered_probelist) return IrmaTaskReturn.success(scan_request.to_dict()) except Exception as e: log.exception(e) raise
def scan(file, probelist, frontend_scan): try: with session_transaction() as session: log.debug("scan_id: %s fileweb_id: %s received %s", frontend_scan, file, probelist) user = User.get_by_rmqvhost(session) scan = scan_ctrl.new(frontend_scan, user, session) # Now, create one subtask per file to scan per probe new_jobs = [] for probe in probelist: if probe in probe_ctrl.available_probes: j = Job(scan.id, file, probe) session.add(j) new_jobs.append(j) else: # send an error message to not stuck the scan # One probe asked is no more present res = probe_ctrl.create_error_results( probe, "missing probe", session) celery_frontend.scan_result(file, probe, res) session.commit() scan_ctrl.launch(scan, new_jobs, session) log.info( "scan_id %s: file %s received / %d active probe(s) / " "%d job(s) launched", scan.scan_id, file, len(probe_ctrl.available_probes), len(scan.jobs)) except Exception as e: log.exception(type(e).__name__ + " : " + str(e)) raise
def scan(frontend_scanid, scan_request_dict): try: with session_transaction() as session: log.debug("scanid: %s received %s", frontend_scanid, scan_request_dict) user = User.get_by_rmqvhost(session) scan_request = IrmaScanRequest(scan_request_dict) scan = scan_ctrl.new(frontend_scanid, user, scan_request.nb_files, session) available_probelist = probe_ctrl.get_list(session) # Now, create one subtask per file to # scan per probe new_jobs = [] for filehash in scan_request.filehashes(): probelist = scan_request.get_probelist(filehash) scan_ctrl.check_probelist(scan, probelist, available_probelist, session) for probe in probelist: j = Job(scan.id, filehash, probe) session.add(j) new_jobs.append(j) session.commit() scan_ctrl.launch(scan, new_jobs, session) celery_frontend.scan_launched(scan.scan_id, scan_request.to_dict()) log.info( "scanid %s: %d file(s) received / %d active probe(s) / " "%d job(s) launched", scan.scan_id, scan.nb_files, len(available_probelist), len(scan.jobs)) except Exception as e: log.exception(e) raise
def probe_list_refresh(): try: with session_transaction() as session: probe_ctrl.refresh_probelist(session) except Exception as e: log.exception(e) return IrmaTaskReturn.error("Error getting probelist")
def mimetype_filter_scan_request(scan_request_dict): try: with session_transaction() as session: available_probelist = probe_ctrl.get_list(session) scan_request = IrmaScanRequest(scan_request_dict) for filehash in scan_request.filehashes(): filtered_probelist = [] mimetype = scan_request.get_mimetype(filehash) mimetype_probelist = probe_ctrl.mimetype_probelist(mimetype, session) probe_list = scan_request.get_probelist(filehash) log.debug("filehash: %s probe_list: %s", filehash, "-".join(probe_list)) # first check probe_list for unknown probe for probe in probe_list: # check if probe exists if probe not in available_probelist: log.warning("probe %s not available", probe) if probe in mimetype_probelist: # probe is asked and supported by mimetype log.debug("filehash %s probe %s asked" + " and supported for %s append to request", filehash, probe, mimetype) filtered_probelist.append(probe) # update probe list in scan request scan_request.set_probelist(filehash, filtered_probelist) return IrmaTaskReturn.success(scan_request.to_dict()) except Exception as e: log.exception(e) return IrmaTaskReturn.error("Brain error")
def probe_list_refresh(): try: with session_transaction() as session: probe_ctrl.refresh_probelist(session) except Exception as e: log.exception(type(e).__name__ + " : " + str(e)) return IrmaTaskReturn.error("Error getting probelist")
def test001_transaction(self, m_SQLDatabase): m_db_session = MagicMock() m_SQLDatabase.get_session.return_value = m_db_session with module.session_transaction(): pass m_db_session.commit.assert_called() m_db_session.rollback.assert_not_called() m_db_session.close.assert_called()
def probe_list(): try: with session_transaction() as session: probe_list = probe_ctrl.get_list(session) return IrmaTaskReturn.success(probe_list) except Exception as e: log.exception(e) return IrmaTaskReturn.error("Error getting probelist")
def _finish(job_id, status): with session_transaction() as session: job = Job.load(job_id, session) job.status = status job.ts_end = timestamp() job.update(['status', 'ts_end'], session) scan_id = job.scan.id scanctrl.check_finished(scan_id)
def flush(scan_id): with session_transaction() as session: scan = Scan.load(scan_id, session) if scan.status == IrmaScanStatus.flushed: return for job in scan.jobs: session.delete(job) scan.status = IrmaScanStatus.flushed
def test002_transaction_error(self, m_session): exception = IrmaDatabaseError with self.assertRaises(exception): with module.session_transaction(): raise exception m_session.commit.assert_not_called() m_session.rollback.assert_called() m_session.close.assert_called()
def check_finished(scan_id): with session_transaction() as session: scan = Scan.load(scan_id, session) if scan.status == IrmaScanStatus.processed: return True if scan.finished(): scan.status = IrmaScanStatus.processed return True return False
def register_probe(name, display_name, category, mimetype_filter): try: log.info("probe %s category %s registered [%s] transfer to scan_app", name, category, mimetype_filter) with session_transaction() as session: probe_ctrl.register(name, display_name, category, mimetype_filter, session) except Exception as e: log.exception(e) raise register_probe.retry(countdown=5, max_retries=3, exc=e)
def scan_cancel(frontend_scan_id): try: with session_transaction() as session: user = User.get_by_rmqvhost(session) scan = Scan.get_scan(frontend_scan_id, user.id, session) res = scan_ctrl.cancel(scan, session) return IrmaTaskReturn.success(res) except Exception as e: log.exception(e) return IrmaTaskReturn.error("cancel error on brain")
def test002_transaction_error(self, m_SQLDatabase): m_db_session = MagicMock() m_SQLDatabase.get_session.return_value = m_db_session exception = IrmaDatabaseError with self.assertRaises(exception): with module.session_transaction(): raise exception m_db_session.commit.assert_not_called() m_db_session.rollback.assert_called() m_db_session.close.assert_called()
def scan_flush(scan_id): try: log.info("scan %s: flush requested", scan_id) with session_transaction() as session: user = User.get_by_rmqvhost(session) scan = Scan.get_scan(scan_id, user.id, session) scan_ctrl.flush(scan, session) except Exception as e: log.exception(type(e).__name__ + " : " + str(e)) return
def scan_flush(frontend_scan_id): try: log.info("scan_id %s: scan flush requested", frontend_scan_id) with session_transaction() as session: user = User.get_by_rmqvhost(session) scan = Scan.get_scan(frontend_scan_id, user.id, session) scan_ctrl.flush(scan, session) except Exception as e: log.exception(e) return
def scan_flush(scan_id): try: log.info("scan %s: flush requested", scan_id) with session_transaction() as session: user = User.get_by_rmqvhost(session) scan = Scan.get_scan(scan_id, user.id, session) scan_ctrl.flush(scan, session) except Exception as e: log.exception(e) return
def files_flush(files, scan_id): try: with session_transaction() as session: user = User.get_by_rmqvhost(session) scan = Scan.get_scan(scan_id, user.id, session) ftpuser = scan.user.ftpuser log.debug("Flushing files %s", files) ftp_ctrl.flush(ftpuser, files) except Exception as e: log.exception(e) return
def scan_cancel(scan_id): try: log.info("scan %s: cancel", scan_id) with session_transaction() as session: user = User.get_by_rmqvhost(session) scan = Scan.get_scan(scan_id, user.id, session) res = scan_ctrl.cancel(scan, session) return IrmaTaskReturn.success(res) except Exception as e: log.exception(type(e).__name__ + " : " + str(e)) return IrmaTaskReturn.error("cancel error on brain")
def files_flush(files, scan_id): try: with session_transaction() as session: user = User.get_by_rmqvhost(session) scan = Scan.get_scan(scan_id, user.id, session) ftpuser = scan.user.ftpuser log.debug("Flushing files %s", files) ftp_ctrl.flush(ftpuser, files) except Exception as e: log.exception(type(e).__name__ + " : " + str(e)) return
def setUp(self): self.user_name = "test_user" self.user_rmqvhost = "test_vhost" self.user_ftpuser = "******" self.user_quota = 100 self.scanid = UUID.generate() try: self.userid = user_ctrl.get_userid(self.user_rmqvhost) except IrmaDatabaseResultNotFound: # user does not exist create one with session_transaction() as session: user = User(self.user_name, self.user_rmqvhost, self.user_ftpuser, self.user_quota) user.save(session) session.commit() self.userid = user.id
if len(sys.argv) not in (4, 5): print("usage: {0} <username> <rmqvhost> <ftpuser> [quota]\n" " with <username> a string\n" " <rmqvhost> the rmqvhost used for the frontend\n" " <ftpuser> the ftpuser used by the frontend\n" " [quota] the number of file scan quota\n" "example: {0} test1 mqfrontend frontend" "".format(sys.argv[0])) sys.exit(1) name = sys.argv[1] rmqvhost = sys.argv[2] ftpuser = sys.argv[3] # quota is in number of files (0 means disabled) quota = int(sys.argv[4]) if len(sys.argv) == 5 else 0 with session_transaction() as session: try: user = User.get_by_rmqvhost(rmqvhost, session) print("rmqvhost {0} is already assigned to user {1}. " "Updating with new parameters.".format(user.name, user.rmqvhost)) user = user.load(user.id, session) user.name = name user.ftpuser = ftpuser user.quota = quota user.update(['name', 'ftpuser', 'quota'], session) except IrmaDatabaseResultNotFound: user = User(name=name, rmqvhost=rmqvhost, ftpuser=ftpuser, quota=quota) user.save(session)
print("SQL directory does not exist {0}" "..creating".format(dirname)) os.makedirs(dirname) os.chmod(dirname, 0o777) elif not (os.path.isdir(dirname)): print("Error. SQL directory is a not a dir {0}" "".format(dirname)) raise IrmaDatabaseError("Can not create Brain database dir") if not os.path.exists(db_name): # touch like method to create a rw-rw-rw- file for db open(db_name, 'a').close() os.chmod(db_name, 0o666) # Retrieve database informations url = config.get_sql_url() engine = create_engine(url, echo=config.sql_debug_enabled()) # and create Database in case Base.metadata.create_all(engine) with session_transaction() as session: try: user = User.get_by_rmqvhost(session, rmqvhost=rmqvhost) print("rmqvhost {0} is already assigned to user {1}. " "Updating with new parameters.".format(user.name, user.rmqvhost)) user = user.load(user.id, session) user.name = name user.ftpuser = ftpuser user.update(['name', 'ftpuser'], session) except IrmaDatabaseResultNotFound: user = User(name=name, rmqvhost=rmqvhost, ftpuser=ftpuser) user.save(session)
def error(scan_id, code): with session_transaction() as session: scan = Scan.load(scan_id, session) scan.status = code
def new(frontend_scan_id, user_id, nb_files): with session_transaction() as session: scan = Scan(frontend_scan_id, user_id, nb_files) scan.save(session) session.commit() return scan.id
def test001_transaction(self, m_session): with module.session_transaction(): pass m_session.commit.assert_called() m_session.rollback.assert_not_called() m_session.close.assert_called()
def set_taskid(job_id, task_id): with session_transaction() as session: job = Job.load(job_id, session) job.task_id = task_id
def _set_status(scan_id, code): with session_transaction() as session: scan = Scan.load(scan_id, session) scan.status = code session.commit()
def new(scan_id, filename, probe, task_id): with session_transaction() as session: j = Job(filename, probe, scan_id, task_id) j.save(session) session.commit() return j.id