def scan(frontend_scanid, scan_request_dict):
try:
with session_transaction() as session:
log.debug("scanid: %s received %s", frontend_scanid,
scan_request_dict)
user = User.get_by_rmqvhost(session)
scan_request = IrmaScanRequest(scan_request_dict)
scan = scan_ctrl.new(frontend_scanid, user, scan_request.nb_files,
session)
available_probelist = probe_ctrl.get_list(session)
# Now, create one subtask per file to
# scan per probe
new_jobs = []
for filehash in scan_request.filehashes():
probelist = scan_request.get_probelist(filehash)
scan_ctrl.check_probelist(scan, probelist, available_probelist,
session)
for probe in probelist:
j = Job(scan.id, filehash, probe)
session.add(j)
new_jobs.append(j)
session.commit()
scan_ctrl.launch(scan, new_jobs, session)
celery_frontend.scan_launched(scan.scan_id, scan_request.to_dict())
log.info(
"scanid %s: %d file(s) received / %d active probe(s) / "
"%d job(s) launched", scan.scan_id, scan.nb_files,
len(available_probelist), len(scan.jobs))
except Exception as e:
log.exception(e)
raise
def scan(file, probelist, frontend_scan):
try:
with session_transaction() as session:
log.debug("scan_id: %s fileweb_id: %s received %s", frontend_scan,
file, probelist)
user = User.get_by_rmqvhost(session)
scan = scan_ctrl.new(frontend_scan, user, session)
# Now, create one subtask per file to scan per probe
new_jobs = []
for probe in probelist:
if probe in probe_ctrl.available_probes:
j = Job(scan.id, file, probe)
session.add(j)
new_jobs.append(j)
else:
# send an error message to not stuck the scan
# One probe asked is no more present
res = probe_ctrl.create_error_results(
probe, "missing probe", session)
celery_frontend.scan_result(file, probe, res)
session.commit()
scan_ctrl.launch(scan, new_jobs, session)
log.info(
"scan_id %s: file %s received / %d active probe(s) / "
"%d job(s) launched", scan.scan_id, file,
len(probe_ctrl.available_probes), len(scan.jobs))
except Exception as e:
log.exception(type(e).__name__ + " : " + str(e))
raise
def test_scan_cancel(self):
scanid = scan_ctrl.new(self.scanid, self.userid, 10)
for code in IrmaScanStatus.label.keys():
scan_ctrl.error(scanid, code)
with session_query() as session:
scan = Scan.load(scanid, session)
self.assertEqual(scan.status, code)
def scan(file, probelist, frontend_scan):
try:
with session_transaction() as session:
log.debug("scan_id: %s fileweb_id: %s received %s", frontend_scan,
file, probelist)
user = User.get_by_rmqvhost(session)
scan = scan_ctrl.new(frontend_scan, user, session)
# Now, create one subtask per file to scan per probe
new_jobs = []
for probe in probelist:
if probe in probe_ctrl.available_probes:
j = Job(scan.id, file, probe)
session.add(j)
new_jobs.append(j)
else:
# send an error message to not stuck the scan
# One probe asked is no more present
res = probe_ctrl.create_error_results(probe,
"missing probe",
session)
celery_frontend.scan_result(file,
probe,
res)
session.commit()
scan_ctrl.launch(scan, new_jobs, session)
log.info("scan_id %s: file %s received / %d active probe(s) / "
"%d job(s) launched",
scan.scan_id,
file,
len(probe_ctrl.available_probes),
len(scan.jobs))
except Exception as e:
log.exception(e)
raise
def scan(frontend_scanid, scan_request_dict):
try:
with session_transaction() as session:
log.debug("scanid: %s received %s", frontend_scanid,
scan_request_dict)
user = User.get_by_rmqvhost(session)
scan_request = IrmaScanRequest(scan_request_dict)
scan = scan_ctrl.new(frontend_scanid, user, scan_request.nb_files,
session)
available_probelist = probe_ctrl.get_list(session)
# Now, create one subtask per file to
# scan per probe
new_jobs = []
for filehash in scan_request.filehashes():
probelist = scan_request.get_probelist(filehash)
scan_ctrl.check_probelist(scan, probelist,
available_probelist, session)
for probe in probelist:
j = Job(scan.id, filehash, probe)
session.add(j)
new_jobs.append(j)
session.commit()
scan_ctrl.launch(scan, new_jobs, session)
celery_frontend.scan_launched(scan.scan_id,
scan_request.to_dict())
log.info("scanid %s: %d file(s) received / %d active probe(s) / "
"%d job(s) launched",
scan.scan_id,
scan.nb_files,
len(available_probelist),
len(scan.jobs))
except Exception as e:
log.exception(e)
raise
def test_scan_new_status(self):
# test we have corrects fields
scanid = scan_ctrl.new(self.scanid, self.userid, 10)
with session_query() as session:
scan = Scan.load(scanid, session)
self.assertIsNotNone(scan.timestamp)
self.assertEqual(scan.status, IrmaScanStatus.empty)
self.assertEqual(scan.nb_files, 10)
self.assertIsNotNone(scan.nb_jobs)
def test002_new_file_not_existing(self, m_scan):
m_scan.get_scan.side_effect = IrmaDatabaseResultNotFound()
old_nb_files = self.scan.nb_files
new_nb_files = randint(150, 200)
scan = module.new(self.frontend_scanid, self.user, new_nb_files,
self.session)
self.session.query().filter().update.assert_not_called()
m_scan().save.assert_called()
self.assertNotEqual(self.nb_files, new_nb_files)
self.assertEqual(scan.nb_files, m_scan().nb_files)
def test_scan_launched(self):
scan_id = scan_ctrl.new(self.scanid, self.userid, 10)
for i in xrange(0, 10):
for probe in ['probe1', 'probe2']:
job_ctrl.new(scan_id, "file-{0}".format(i), probe, 1)
scan_ctrl.launched(scan_id)
with session_query() as session:
scan = Scan.load(scan_id, session)
self.assertEqual(scan.status, IrmaScanStatus.launched)
self.assertEqual(scan.nb_files, 10)
self.assertEqual(scan.nb_jobs, 20)
def test001_new_scan_existing(self, m_scan):
m_scan.get_scan.return_value = self.scan
old_nb_files = self.scan.nb_files
new_nb_files = randint(150, 200)
scan = module.new(self.frontend_scanid, self.user, new_nb_files,
self.session)
self.session.query().filter().update.assert_called()
m_scan().save.assert_not_called()
self.assertNotEqual(self.nb_files, new_nb_files)
self.assertEqual(scan, self.scan)
self.assertEqual(scan.nb_files, new_nb_files + old_nb_files)
def test_user_quota_update(self):
# test update quota
(before, _) = user_ctrl.get_quota(self.userid)
scanid = scan_ctrl.new(self.scanid, self.userid, 2)
for i in xrange(0, 10):
for probe in ['probe1', 'probe2']:
job_ctrl.new(scanid, "file-{0}".format(i), probe, 1)
scan_ctrl.launched(scanid)
(after, _) = user_ctrl.get_quota(self.userid)
self.assertEqual(before - after, 20)
with session_query() as session:
user = User.load(self.userid, session)
scan_ids = [scan.id for scan in user.scans]
self.assertNotEqual(scan_ids, [])
self.assertTrue(scanid in scan_ids)
def test_scan_progress(self):
scan_id = scan_ctrl.new(self.scanid, self.userid, 10)
job_ids = []
for i in xrange(0, 10):
for probe in ['probe1', 'probe2']:
job_ids.append(job_ctrl.new(scan_id, "file-{0}".format(i),
probe, 1))
scan_ctrl.launched(scan_id)
shuffle(job_ids)
for i, job_id in enumerate(job_ids[:-1]):
job_ctrl.success(job_id)
(status, progress_details) = scan_ctrl.progress(scan_id)
self.assertEqual(status,
IrmaScanStatus.label[IrmaScanStatus.launched])
self.assertIsNotNone(progress_details)
self.assertEqual(progress_details['successful'], i + 1)
def test_scan_job_error(self):
scan_id = scan_ctrl.new(self.scanid, self.userid, 10)
job_ids = []
for i in xrange(0, 10):
for probe in ['probe1', 'probe2']:
job_ids.append(job_ctrl.new(scan_id, "file-{0}".format(i),
probe, 1))
scan_ctrl.launched(scan_id)
shuffle(job_ids)
for job_id in job_ids:
job_ctrl.error(job_id)
self.assertTrue(scan_ctrl.check_finished(scan_id))
with session_query() as session:
scan = Scan.load(scan_id, session)
self.assertEqual(scan.status, IrmaScanStatus.processed)
self.assertEqual(scan.nb_files, 10)
self.assertEqual(scan.nb_jobs, 20)
def test_new_file_not_existing(self, m_scan):
m_scan.get_scan.side_effect = IrmaDatabaseResultNotFound()
module.new(self.frontend_scanid, self.user, self.session)
self.session.add.assert_called()
def test_new_file_not_existing(self, m_scan):
m_scan.get_scan.side_effect = IrmaDatabaseResultNotFound()
module.new(self.frontend_scanid, self.user, self.session)
self.session.add.assert_called()
def test_new_scan_existing(self, m_scan):
m_scan.get_scan.return_value = self.scan
scan = module.new(self.frontend_scanid, self.user, self.session)
self.session.add.assert_not_called()
self.assertEqual(scan, self.scan)
def scan(frontend_scanid, scan_request):
try:
log.info("%s", frontend_scanid)
# TODO user should be identified by RMQ vhost
# read vhost from config as workaround
rmqvhost = config.get_frontend_rmqvhost()
user_id = user_ctrl.get_userid(rmqvhost)
ftpuser = user_ctrl.get_ftpuser(user_id)
# create an initial entry to track future errors
# tell frontend that frontend_scanid is now known to brain
# progress available
scan_id = scan_ctrl.new(frontend_scanid, user_id, len(scan_request))
# send a scan launched event to frontend
(remaining, quota) = user_ctrl.get_quota(user_id)
if remaining is not None:
log.info("%s quota remaining {0}/{1}",
frontend_scanid,
remaining,
quota)
else:
log.info("%s unlimited quota",
frontend_scanid)
if remaining <= 0:
scan_ctrl.error(scan_id, IrmaScanStatus.error)
available_probelist = celery_probe.get_probelist()
for (filename, probe_list) in scan_request.items():
if probe_list is None:
scan_ctrl.error(scan_id, IrmaScanStatus.error_probe_missing)
log.info("%s Empty probe list", frontend_scanid)
return IrmaTaskReturn.error("empty probe list on brain")
# first check probe_list
for p in probe_list:
# check if probe exists
if p not in available_probelist:
scan_ctrl.error(scan_id,
IrmaScanStatus.error_probe_missing)
msg = "Unknown probe {0}".format(p)
log.info("%s Unknown probe %s",
frontend_scanid,
p)
return IrmaTaskReturn.error(msg)
# Now, create one subtask per file to
# scan per probe according to quota
for probe in probe_list:
if remaining is not None:
if remaining <= 0:
break
else:
remaining -= 1
task_id = UUID.generate()
job_id = job_ctrl.new(scan_id, filename, probe, task_id)
celery_probe.job_launch(ftpuser,
frontend_scanid,
filename,
probe,
job_id,
task_id)
scan_ctrl.launched(scan_id)
celery_frontend.scan_launched(frontend_scanid)
log.info(
"%s %d files receives / %d active probe /" +
" %d probe used / %d jobs launched",
frontend_scanid,
len(scan_request),
len(available_probelist),
len(probe_list),
scan_ctrl.get_nbjobs(scan_id))
except:
log.info("exception", exc_info=True)
raise
def test_new_scan_existing(self, m_scan):
m_scan.get_scan.return_value = self.scan
scan = module.new(self.frontend_scanid, self.user, self.session)
self.session.add.assert_not_called()
self.assertEqual(scan, self.scan)
def test_scan_new_id(self):
# test we got an id
scanid = scan_ctrl.new(self.scanid, self.userid, 0)
self.assertIsNotNone(scanid)