def mimetype_filter_scan_request(scan_request_dict):
try:
with session_transaction() as session:
scan_request = IrmaScanRequest(scan_request_dict)
for file in scan_request.files():
filtered_probelist = []
mimetype = scan_request.get_mimetype(file)
mimetype_probelist = probe_ctrl.mimetype_probelist(mimetype,
session)
probe_list = scan_request.get_probelist(file)
log.debug("file: %s probe_list: %s",
file, "-".join(probe_list))
# first check probe_list for unknown probe
for probe in probe_list:
# check if probe exists
if probe not in probe_ctrl.available_probes:
log.warning("probe %s not available", probe)
if probe in mimetype_probelist:
# probe is asked and supported by mimetype
log.debug("file %s probe %s asked" +
" and supported for %s append to request",
file, probe, mimetype)
filtered_probelist.append(probe)
# update probe list in scan request
scan_request.set_probelist(file, filtered_probelist)
return IrmaTaskReturn.success(scan_request.to_dict())
except Exception as e:
log.exception(e)
return IrmaTaskReturn.error("Brain error")
def scan(frontend_scanid, scan_request_dict):
try:
with session_transaction() as session:
log.debug("scanid: %s received %s", frontend_scanid,
scan_request_dict)
user = User.get_by_rmqvhost(session)
scan_request = IrmaScanRequest(scan_request_dict)
scan = scan_ctrl.new(frontend_scanid, user, scan_request.nb_files,
session)
available_probelist = probe_ctrl.get_list(session)
# Now, create one subtask per file to
# scan per probe
new_jobs = []
for filehash in scan_request.filehashes():
probelist = scan_request.get_probelist(filehash)
scan_ctrl.check_probelist(scan, probelist,
available_probelist, session)
for probe in probelist:
j = Job(scan.id, filehash, probe)
session.add(j)
new_jobs.append(j)
session.commit()
scan_ctrl.launch(scan, new_jobs, session)
celery_frontend.scan_launched(scan.scan_id,
scan_request.to_dict())
log.info("scanid %s: %d file(s) received / %d active probe(s) / "
"%d job(s) launched",
scan.scan_id,
scan.nb_files,
len(available_probelist),
len(scan.jobs))
except Exception as e:
log.exception(e)
raise
def scan(file, probelist, frontend_scan):
try:
with session_transaction() as session:
log.debug("scan_id: %s fileweb_id: %s received %s", frontend_scan,
file, probelist)
user = User.get_by_rmqvhost(session)
scan = scan_ctrl.new(frontend_scan, user, session)
# Now, create one subtask per file to scan per probe
new_jobs = []
for probe in probelist:
if probe in probe_ctrl.available_probes:
j = Job(scan.id, file, probe)
session.add(j)
new_jobs.append(j)
else:
# send an error message to not stuck the scan
# One probe asked is no more present
res = probe_ctrl.create_error_results(probe,
"missing probe",
session)
celery_frontend.scan_result(file,
probe,
res)
session.commit()
scan_ctrl.launch(scan, new_jobs, session)
log.info("scan_id %s: file %s received / %d active probe(s) / "
"%d job(s) launched",
scan.scan_id,
file,
len(probe_ctrl.available_probes),
len(scan.jobs))
except Exception as e:
log.exception(e)
raise
def mimetype_filter_scan_request(scan_request_dict):
try:
with session_transaction() as session:
available_probelist = probe_ctrl.get_list(session)
scan_request = IrmaScanRequest(scan_request_dict)
for filehash in scan_request.filehashes():
filtered_probelist = []
mimetype = scan_request.get_mimetype(filehash)
mimetype_probelist = probe_ctrl.mimetype_probelist(mimetype,
session)
probe_list = scan_request.get_probelist(filehash)
log.debug("filehash: %s probe_list: %s",
filehash, "-".join(probe_list))
# first check probe_list for unknown probe
for probe in probe_list:
# check if probe exists
if probe not in available_probelist:
msg = "Unknown probe {0}".format(probe)
log.error(msg)
return IrmaTaskReturn.error(msg)
if probe in mimetype_probelist:
# probe is asked but not supported by mimetype
log.debug("filehash %s probe %s asked but" +
" not supported for %s",
filehash, probe, mimetype)
filtered_probelist.append(probe)
# update probe list in scan request
scan_request.set_probelist(filehash, filtered_probelist)
return IrmaTaskReturn.success(scan_request.to_dict())
except Exception as e:
log.exception(e)
raise
def scan(file, probelist, frontend_scan):
try:
with session_transaction() as session:
log.debug("scan_id: %s fileweb_id: %s received %s", frontend_scan,
file, probelist)
user = User.get_by_rmqvhost(session)
scan = scan_ctrl.new(frontend_scan, user, session)
# Now, create one subtask per file to scan per probe
new_jobs = []
for probe in probelist:
if probe in probe_ctrl.available_probes:
j = Job(scan.id, file, probe)
session.add(j)
new_jobs.append(j)
else:
# send an error message to not stuck the scan
# One probe asked is no more present
res = probe_ctrl.create_error_results(
probe, "missing probe", session)
celery_frontend.scan_result(file, probe, res)
session.commit()
scan_ctrl.launch(scan, new_jobs, session)
log.info(
"scan_id %s: file %s received / %d active probe(s) / "
"%d job(s) launched", scan.scan_id, file,
len(probe_ctrl.available_probes), len(scan.jobs))
except Exception as e:
log.exception(type(e).__name__ + " : " + str(e))
raise
def scan(frontend_scanid, scan_request_dict):
try:
with session_transaction() as session:
log.debug("scanid: %s received %s", frontend_scanid,
scan_request_dict)
user = User.get_by_rmqvhost(session)
scan_request = IrmaScanRequest(scan_request_dict)
scan = scan_ctrl.new(frontend_scanid, user, scan_request.nb_files,
session)
available_probelist = probe_ctrl.get_list(session)
# Now, create one subtask per file to
# scan per probe
new_jobs = []
for filehash in scan_request.filehashes():
probelist = scan_request.get_probelist(filehash)
scan_ctrl.check_probelist(scan, probelist, available_probelist,
session)
for probe in probelist:
j = Job(scan.id, filehash, probe)
session.add(j)
new_jobs.append(j)
session.commit()
scan_ctrl.launch(scan, new_jobs, session)
celery_frontend.scan_launched(scan.scan_id, scan_request.to_dict())
log.info(
"scanid %s: %d file(s) received / %d active probe(s) / "
"%d job(s) launched", scan.scan_id, scan.nb_files,
len(available_probelist), len(scan.jobs))
except Exception as e:
log.exception(e)
raise
def probe_list_refresh():
try:
with session_transaction() as session:
probe_ctrl.refresh_probelist(session)
except Exception as e:
log.exception(e)
return IrmaTaskReturn.error("Error getting probelist")
def mimetype_filter_scan_request(scan_request_dict):
try:
with session_transaction() as session:
available_probelist = probe_ctrl.get_list(session)
scan_request = IrmaScanRequest(scan_request_dict)
for filehash in scan_request.filehashes():
filtered_probelist = []
mimetype = scan_request.get_mimetype(filehash)
mimetype_probelist = probe_ctrl.mimetype_probelist(mimetype,
session)
probe_list = scan_request.get_probelist(filehash)
log.debug("filehash: %s probe_list: %s",
filehash, "-".join(probe_list))
# first check probe_list for unknown probe
for probe in probe_list:
# check if probe exists
if probe not in available_probelist:
log.warning("probe %s not available", probe)
if probe in mimetype_probelist:
# probe is asked and supported by mimetype
log.debug("filehash %s probe %s asked" +
" and supported for %s append to request",
filehash, probe, mimetype)
filtered_probelist.append(probe)
# update probe list in scan request
scan_request.set_probelist(filehash, filtered_probelist)
return IrmaTaskReturn.success(scan_request.to_dict())
except Exception as e:
log.exception(e)
return IrmaTaskReturn.error("Brain error")
def probe_list_refresh():
try:
with session_transaction() as session:
probe_ctrl.refresh_probelist(session)
except Exception as e:
log.exception(type(e).__name__ + " : " + str(e))
return IrmaTaskReturn.error("Error getting probelist")
def test001_transaction(self, m_SQLDatabase):
m_db_session = MagicMock()
m_SQLDatabase.get_session.return_value = m_db_session
with module.session_transaction():
pass
m_db_session.commit.assert_called()
m_db_session.rollback.assert_not_called()
m_db_session.close.assert_called()
def probe_list():
try:
with session_transaction() as session:
probe_list = probe_ctrl.get_list(session)
return IrmaTaskReturn.success(probe_list)
except Exception as e:
log.exception(e)
return IrmaTaskReturn.error("Error getting probelist")
def _finish(job_id, status):
with session_transaction() as session:
job = Job.load(job_id, session)
job.status = status
job.ts_end = timestamp()
job.update(['status', 'ts_end'], session)
scan_id = job.scan.id
scanctrl.check_finished(scan_id)
def probe_list():
try:
with session_transaction() as session:
probe_list = probe_ctrl.get_list(session)
return IrmaTaskReturn.success(probe_list)
except Exception as e:
log.exception(e)
return IrmaTaskReturn.error("Error getting probelist")
def flush(scan_id):
with session_transaction() as session:
scan = Scan.load(scan_id, session)
if scan.status == IrmaScanStatus.flushed:
return
for job in scan.jobs:
session.delete(job)
scan.status = IrmaScanStatus.flushed
def test002_transaction_error(self, m_session):
exception = IrmaDatabaseError
with self.assertRaises(exception):
with module.session_transaction():
raise exception
m_session.commit.assert_not_called()
m_session.rollback.assert_called()
m_session.close.assert_called()
def test001_transaction(self, m_SQLDatabase):
m_db_session = MagicMock()
m_SQLDatabase.get_session.return_value = m_db_session
with module.session_transaction():
pass
m_db_session.commit.assert_called()
m_db_session.rollback.assert_not_called()
m_db_session.close.assert_called()
def check_finished(scan_id):
with session_transaction() as session:
scan = Scan.load(scan_id, session)
if scan.status == IrmaScanStatus.processed:
return True
if scan.finished():
scan.status = IrmaScanStatus.processed
return True
return False
def register_probe(name, display_name, category, mimetype_filter):
try:
log.info("probe %s category %s registered [%s] transfer to scan_app",
name, category, mimetype_filter)
with session_transaction() as session:
probe_ctrl.register(name, display_name, category,
mimetype_filter, session)
except Exception as e:
log.exception(e)
raise register_probe.retry(countdown=5, max_retries=3, exc=e)
def scan_cancel(frontend_scan_id):
try:
with session_transaction() as session:
user = User.get_by_rmqvhost(session)
scan = Scan.get_scan(frontend_scan_id, user.id, session)
res = scan_ctrl.cancel(scan, session)
return IrmaTaskReturn.success(res)
except Exception as e:
log.exception(e)
return IrmaTaskReturn.error("cancel error on brain")
def scan_cancel(frontend_scan_id):
try:
with session_transaction() as session:
user = User.get_by_rmqvhost(session)
scan = Scan.get_scan(frontend_scan_id, user.id, session)
res = scan_ctrl.cancel(scan, session)
return IrmaTaskReturn.success(res)
except Exception as e:
log.exception(e)
return IrmaTaskReturn.error("cancel error on brain")
def test002_transaction_error(self, m_SQLDatabase):
m_db_session = MagicMock()
m_SQLDatabase.get_session.return_value = m_db_session
exception = IrmaDatabaseError
with self.assertRaises(exception):
with module.session_transaction():
raise exception
m_db_session.commit.assert_not_called()
m_db_session.rollback.assert_called()
m_db_session.close.assert_called()
def scan_flush(scan_id):
try:
log.info("scan %s: flush requested", scan_id)
with session_transaction() as session:
user = User.get_by_rmqvhost(session)
scan = Scan.get_scan(scan_id, user.id, session)
scan_ctrl.flush(scan, session)
except Exception as e:
log.exception(type(e).__name__ + " : " + str(e))
return
def scan_flush(frontend_scan_id):
try:
log.info("scan_id %s: scan flush requested", frontend_scan_id)
with session_transaction() as session:
user = User.get_by_rmqvhost(session)
scan = Scan.get_scan(frontend_scan_id, user.id, session)
scan_ctrl.flush(scan, session)
except Exception as e:
log.exception(e)
return
def register_probe(name, display_name, category, mimetype_filter):
try:
log.info("probe %s category %s registered [%s] transfer to scan_app",
name, category, mimetype_filter)
with session_transaction() as session:
probe_ctrl.register(name, display_name, category,
mimetype_filter, session)
except Exception as e:
log.exception(e)
raise register_probe.retry(countdown=5, max_retries=3, exc=e)
def scan_flush(scan_id):
try:
log.info("scan %s: flush requested", scan_id)
with session_transaction() as session:
user = User.get_by_rmqvhost(session)
scan = Scan.get_scan(scan_id, user.id, session)
scan_ctrl.flush(scan, session)
except Exception as e:
log.exception(e)
return
def test002_transaction_error(self, m_SQLDatabase):
m_db_session = MagicMock()
m_SQLDatabase.get_session.return_value = m_db_session
exception = IrmaDatabaseError
with self.assertRaises(exception):
with module.session_transaction():
raise exception
m_db_session.commit.assert_not_called()
m_db_session.rollback.assert_called()
m_db_session.close.assert_called()
def files_flush(files, scan_id):
try:
with session_transaction() as session:
user = User.get_by_rmqvhost(session)
scan = Scan.get_scan(scan_id, user.id, session)
ftpuser = scan.user.ftpuser
log.debug("Flushing files %s", files)
ftp_ctrl.flush(ftpuser, files)
except Exception as e:
log.exception(e)
return
def scan_cancel(scan_id):
try:
log.info("scan %s: cancel", scan_id)
with session_transaction() as session:
user = User.get_by_rmqvhost(session)
scan = Scan.get_scan(scan_id, user.id, session)
res = scan_ctrl.cancel(scan, session)
return IrmaTaskReturn.success(res)
except Exception as e:
log.exception(type(e).__name__ + " : " + str(e))
return IrmaTaskReturn.error("cancel error on brain")
def files_flush(files, scan_id):
try:
with session_transaction() as session:
user = User.get_by_rmqvhost(session)
scan = Scan.get_scan(scan_id, user.id, session)
ftpuser = scan.user.ftpuser
log.debug("Flushing files %s", files)
ftp_ctrl.flush(ftpuser, files)
except Exception as e:
log.exception(type(e).__name__ + " : " + str(e))
return
def setUp(self):
self.user_name = "test_user"
self.user_rmqvhost = "test_vhost"
self.user_ftpuser = "******"
self.user_quota = 100
self.scanid = UUID.generate()
try:
self.userid = user_ctrl.get_userid(self.user_rmqvhost)
except IrmaDatabaseResultNotFound:
# user does not exist create one
with session_transaction() as session:
user = User(self.user_name,
self.user_rmqvhost,
self.user_ftpuser,
self.user_quota)
user.save(session)
session.commit()
self.userid = user.id
if len(sys.argv) not in (4, 5):
print("usage: {0} <username> <rmqvhost> <ftpuser> [quota]\n"
" with <username> a string\n"
" <rmqvhost> the rmqvhost used for the frontend\n"
" <ftpuser> the ftpuser used by the frontend\n"
" [quota] the number of file scan quota\n"
"example: {0} test1 mqfrontend frontend"
"".format(sys.argv[0]))
sys.exit(1)
name = sys.argv[1]
rmqvhost = sys.argv[2]
ftpuser = sys.argv[3]
# quota is in number of files (0 means disabled)
quota = int(sys.argv[4]) if len(sys.argv) == 5 else 0
with session_transaction() as session:
try:
user = User.get_by_rmqvhost(rmqvhost, session)
print("rmqvhost {0} is already assigned to user {1}. "
"Updating with new parameters.".format(user.name, user.rmqvhost))
user = user.load(user.id, session)
user.name = name
user.ftpuser = ftpuser
user.quota = quota
user.update(['name', 'ftpuser', 'quota'], session)
except IrmaDatabaseResultNotFound:
user = User(name=name, rmqvhost=rmqvhost, ftpuser=ftpuser, quota=quota)
user.save(session)
print("SQL directory does not exist {0}" "..creating".format(dirname))
os.makedirs(dirname)
os.chmod(dirname, 0o777)
elif not (os.path.isdir(dirname)):
print("Error. SQL directory is a not a dir {0}" "".format(dirname))
raise IrmaDatabaseError("Can not create Brain database dir")
if not os.path.exists(db_name):
# touch like method to create a rw-rw-rw- file for db
open(db_name, 'a').close()
os.chmod(db_name, 0o666)
# Retrieve database informations
url = config.get_sql_url()
engine = create_engine(url, echo=config.sql_debug_enabled())
# and create Database in case
Base.metadata.create_all(engine)
with session_transaction() as session:
try:
user = User.get_by_rmqvhost(session, rmqvhost=rmqvhost)
print("rmqvhost {0} is already assigned to user {1}. "
"Updating with new parameters.".format(user.name, user.rmqvhost))
user = user.load(user.id, session)
user.name = name
user.ftpuser = ftpuser
user.update(['name', 'ftpuser'], session)
except IrmaDatabaseResultNotFound:
user = User(name=name, rmqvhost=rmqvhost, ftpuser=ftpuser)
user.save(session)
def error(scan_id, code):
with session_transaction() as session:
scan = Scan.load(scan_id, session)
scan.status = code
def new(frontend_scan_id, user_id, nb_files):
with session_transaction() as session:
scan = Scan(frontend_scan_id, user_id, nb_files)
scan.save(session)
session.commit()
return scan.id
def test001_transaction(self, m_session):
with module.session_transaction():
pass
m_session.commit.assert_called()
m_session.rollback.assert_not_called()
m_session.close.assert_called()
def set_taskid(job_id, task_id):
with session_transaction() as session:
job = Job.load(job_id, session)
job.task_id = task_id
def _set_status(scan_id, code):
with session_transaction() as session:
scan = Scan.load(scan_id, session)
scan.status = code
session.commit()
def new(scan_id, filename, probe, task_id):
with session_transaction() as session:
j = Job(filename, probe, scan_id, task_id)
j.save(session)
session.commit()
return j.id
