def scan(frontend_scanid, scan_request_dict): try: with session_transaction() as session: log.debug("scanid: %s received %s", frontend_scanid, scan_request_dict) user = User.get_by_rmqvhost(session) scan_request = IrmaScanRequest(scan_request_dict) scan = scan_ctrl.new(frontend_scanid, user, scan_request.nb_files, session) available_probelist = probe_ctrl.get_list(session) # Now, create one subtask per file to # scan per probe new_jobs = [] for filehash in scan_request.filehashes(): probelist = scan_request.get_probelist(filehash) scan_ctrl.check_probelist(scan, probelist, available_probelist, session) for probe in probelist: j = Job(scan.id, filehash, probe) session.add(j) new_jobs.append(j) session.commit() scan_ctrl.launch(scan, new_jobs, session) celery_frontend.scan_launched(scan.scan_id, scan_request.to_dict()) log.info( "scanid %s: %d file(s) received / %d active probe(s) / " "%d job(s) launched", scan.scan_id, scan.nb_files, len(available_probelist), len(scan.jobs)) except Exception as e: log.exception(e) raise
def scan(file, probelist, frontend_scan): try: with session_transaction() as session: log.debug("scan_id: %s fileweb_id: %s received %s", frontend_scan, file, probelist) user = User.get_by_rmqvhost(session) scan = scan_ctrl.new(frontend_scan, user, session) # Now, create one subtask per file to scan per probe new_jobs = [] for probe in probelist: if probe in probe_ctrl.available_probes: j = Job(scan.id, file, probe) session.add(j) new_jobs.append(j) else: # send an error message to not stuck the scan # One probe asked is no more present res = probe_ctrl.create_error_results( probe, "missing probe", session) celery_frontend.scan_result(file, probe, res) session.commit() scan_ctrl.launch(scan, new_jobs, session) log.info( "scan_id %s: file %s received / %d active probe(s) / " "%d job(s) launched", scan.scan_id, file, len(probe_ctrl.available_probes), len(scan.jobs)) except Exception as e: log.exception(type(e).__name__ + " : " + str(e)) raise
def test_scan_cancel(self): scanid = scan_ctrl.new(self.scanid, self.userid, 10) for code in IrmaScanStatus.label.keys(): scan_ctrl.error(scanid, code) with session_query() as session: scan = Scan.load(scanid, session) self.assertEqual(scan.status, code)
def scan(file, probelist, frontend_scan): try: with session_transaction() as session: log.debug("scan_id: %s fileweb_id: %s received %s", frontend_scan, file, probelist) user = User.get_by_rmqvhost(session) scan = scan_ctrl.new(frontend_scan, user, session) # Now, create one subtask per file to scan per probe new_jobs = [] for probe in probelist: if probe in probe_ctrl.available_probes: j = Job(scan.id, file, probe) session.add(j) new_jobs.append(j) else: # send an error message to not stuck the scan # One probe asked is no more present res = probe_ctrl.create_error_results(probe, "missing probe", session) celery_frontend.scan_result(file, probe, res) session.commit() scan_ctrl.launch(scan, new_jobs, session) log.info("scan_id %s: file %s received / %d active probe(s) / " "%d job(s) launched", scan.scan_id, file, len(probe_ctrl.available_probes), len(scan.jobs)) except Exception as e: log.exception(e) raise
def scan(frontend_scanid, scan_request_dict): try: with session_transaction() as session: log.debug("scanid: %s received %s", frontend_scanid, scan_request_dict) user = User.get_by_rmqvhost(session) scan_request = IrmaScanRequest(scan_request_dict) scan = scan_ctrl.new(frontend_scanid, user, scan_request.nb_files, session) available_probelist = probe_ctrl.get_list(session) # Now, create one subtask per file to # scan per probe new_jobs = [] for filehash in scan_request.filehashes(): probelist = scan_request.get_probelist(filehash) scan_ctrl.check_probelist(scan, probelist, available_probelist, session) for probe in probelist: j = Job(scan.id, filehash, probe) session.add(j) new_jobs.append(j) session.commit() scan_ctrl.launch(scan, new_jobs, session) celery_frontend.scan_launched(scan.scan_id, scan_request.to_dict()) log.info("scanid %s: %d file(s) received / %d active probe(s) / " "%d job(s) launched", scan.scan_id, scan.nb_files, len(available_probelist), len(scan.jobs)) except Exception as e: log.exception(e) raise
def test_scan_new_status(self): # test we have corrects fields scanid = scan_ctrl.new(self.scanid, self.userid, 10) with session_query() as session: scan = Scan.load(scanid, session) self.assertIsNotNone(scan.timestamp) self.assertEqual(scan.status, IrmaScanStatus.empty) self.assertEqual(scan.nb_files, 10) self.assertIsNotNone(scan.nb_jobs)
def test002_new_file_not_existing(self, m_scan): m_scan.get_scan.side_effect = IrmaDatabaseResultNotFound() old_nb_files = self.scan.nb_files new_nb_files = randint(150, 200) scan = module.new(self.frontend_scanid, self.user, new_nb_files, self.session) self.session.query().filter().update.assert_not_called() m_scan().save.assert_called() self.assertNotEqual(self.nb_files, new_nb_files) self.assertEqual(scan.nb_files, m_scan().nb_files)
def test_scan_launched(self): scan_id = scan_ctrl.new(self.scanid, self.userid, 10) for i in xrange(0, 10): for probe in ['probe1', 'probe2']: job_ctrl.new(scan_id, "file-{0}".format(i), probe, 1) scan_ctrl.launched(scan_id) with session_query() as session: scan = Scan.load(scan_id, session) self.assertEqual(scan.status, IrmaScanStatus.launched) self.assertEqual(scan.nb_files, 10) self.assertEqual(scan.nb_jobs, 20)
def test001_new_scan_existing(self, m_scan): m_scan.get_scan.return_value = self.scan old_nb_files = self.scan.nb_files new_nb_files = randint(150, 200) scan = module.new(self.frontend_scanid, self.user, new_nb_files, self.session) self.session.query().filter().update.assert_called() m_scan().save.assert_not_called() self.assertNotEqual(self.nb_files, new_nb_files) self.assertEqual(scan, self.scan) self.assertEqual(scan.nb_files, new_nb_files + old_nb_files)
def test_user_quota_update(self): # test update quota (before, _) = user_ctrl.get_quota(self.userid) scanid = scan_ctrl.new(self.scanid, self.userid, 2) for i in xrange(0, 10): for probe in ['probe1', 'probe2']: job_ctrl.new(scanid, "file-{0}".format(i), probe, 1) scan_ctrl.launched(scanid) (after, _) = user_ctrl.get_quota(self.userid) self.assertEqual(before - after, 20) with session_query() as session: user = User.load(self.userid, session) scan_ids = [scan.id for scan in user.scans] self.assertNotEqual(scan_ids, []) self.assertTrue(scanid in scan_ids)
def test_scan_progress(self): scan_id = scan_ctrl.new(self.scanid, self.userid, 10) job_ids = [] for i in xrange(0, 10): for probe in ['probe1', 'probe2']: job_ids.append(job_ctrl.new(scan_id, "file-{0}".format(i), probe, 1)) scan_ctrl.launched(scan_id) shuffle(job_ids) for i, job_id in enumerate(job_ids[:-1]): job_ctrl.success(job_id) (status, progress_details) = scan_ctrl.progress(scan_id) self.assertEqual(status, IrmaScanStatus.label[IrmaScanStatus.launched]) self.assertIsNotNone(progress_details) self.assertEqual(progress_details['successful'], i + 1)
def test_scan_job_error(self): scan_id = scan_ctrl.new(self.scanid, self.userid, 10) job_ids = [] for i in xrange(0, 10): for probe in ['probe1', 'probe2']: job_ids.append(job_ctrl.new(scan_id, "file-{0}".format(i), probe, 1)) scan_ctrl.launched(scan_id) shuffle(job_ids) for job_id in job_ids: job_ctrl.error(job_id) self.assertTrue(scan_ctrl.check_finished(scan_id)) with session_query() as session: scan = Scan.load(scan_id, session) self.assertEqual(scan.status, IrmaScanStatus.processed) self.assertEqual(scan.nb_files, 10) self.assertEqual(scan.nb_jobs, 20)
def test_new_file_not_existing(self, m_scan): m_scan.get_scan.side_effect = IrmaDatabaseResultNotFound() module.new(self.frontend_scanid, self.user, self.session) self.session.add.assert_called()
def test_new_file_not_existing(self, m_scan): m_scan.get_scan.side_effect = IrmaDatabaseResultNotFound() module.new(self.frontend_scanid, self.user, self.session) self.session.add.assert_called()
def test_new_scan_existing(self, m_scan): m_scan.get_scan.return_value = self.scan scan = module.new(self.frontend_scanid, self.user, self.session) self.session.add.assert_not_called() self.assertEqual(scan, self.scan)
def scan(frontend_scanid, scan_request): try: log.info("%s", frontend_scanid) # TODO user should be identified by RMQ vhost # read vhost from config as workaround rmqvhost = config.get_frontend_rmqvhost() user_id = user_ctrl.get_userid(rmqvhost) ftpuser = user_ctrl.get_ftpuser(user_id) # create an initial entry to track future errors # tell frontend that frontend_scanid is now known to brain # progress available scan_id = scan_ctrl.new(frontend_scanid, user_id, len(scan_request)) # send a scan launched event to frontend (remaining, quota) = user_ctrl.get_quota(user_id) if remaining is not None: log.info("%s quota remaining {0}/{1}", frontend_scanid, remaining, quota) else: log.info("%s unlimited quota", frontend_scanid) if remaining <= 0: scan_ctrl.error(scan_id, IrmaScanStatus.error) available_probelist = celery_probe.get_probelist() for (filename, probe_list) in scan_request.items(): if probe_list is None: scan_ctrl.error(scan_id, IrmaScanStatus.error_probe_missing) log.info("%s Empty probe list", frontend_scanid) return IrmaTaskReturn.error("empty probe list on brain") # first check probe_list for p in probe_list: # check if probe exists if p not in available_probelist: scan_ctrl.error(scan_id, IrmaScanStatus.error_probe_missing) msg = "Unknown probe {0}".format(p) log.info("%s Unknown probe %s", frontend_scanid, p) return IrmaTaskReturn.error(msg) # Now, create one subtask per file to # scan per probe according to quota for probe in probe_list: if remaining is not None: if remaining <= 0: break else: remaining -= 1 task_id = UUID.generate() job_id = job_ctrl.new(scan_id, filename, probe, task_id) celery_probe.job_launch(ftpuser, frontend_scanid, filename, probe, job_id, task_id) scan_ctrl.launched(scan_id) celery_frontend.scan_launched(frontend_scanid) log.info( "%s %d files receives / %d active probe /" + " %d probe used / %d jobs launched", frontend_scanid, len(scan_request), len(available_probelist), len(probe_list), scan_ctrl.get_nbjobs(scan_id)) except: log.info("exception", exc_info=True) raise
def test_new_scan_existing(self, m_scan): m_scan.get_scan.return_value = self.scan scan = module.new(self.frontend_scanid, self.user, self.session) self.session.add.assert_not_called() self.assertEqual(scan, self.scan)
def test_scan_new_id(self): # test we got an id scanid = scan_ctrl.new(self.scanid, self.userid, 0) self.assertIsNotNone(scanid)