def test_get_celery_options(self, m_config, m_common_celery_options):
concurrency = randint(0, 50)
soft_time_limit = randint(10, 100)
time_limit = randint(10, 100)
beat_schedule = "whatever"
m_config.celery_options.concurrency = concurrency
m_config.celery_options.soft_time_limit = soft_time_limit
m_config.celery_options.time_limit = time_limit
m_config.celery_options.beat_schedule = beat_schedule
options = module.common_celery_options("app", "app_name", concurrency,
soft_time_limit, time_limit)
options.append("--beat")
options.append("--schedule={}".format(beat_schedule))
self.assertEqual(options, module.get_celery_options("app", "appname"))
def test_get_celery_options(self, m_config,
m_common_celery_options):
concurrency = randint(0, 50)
soft_time_limit = randint(10, 100)
time_limit = randint(10, 100)
beat_schedule = "whatever"
m_config.celery_options.concurrency = concurrency
m_config.celery_options.soft_time_limit = soft_time_limit
m_config.celery_options.time_limit = time_limit
m_config.celery_options.beat_schedule = beat_schedule
options = module.common_celery_options("app",
"app_name",
concurrency,
soft_time_limit,
time_limit)
options.append("--beat")
options.append("--schedule={}".format(beat_schedule))
self.assertEqual(options, module.get_celery_options("app", "appname"))
user = User.get_by_rmqvhost(session)
scan = Scan.get_scan(scan_id, user.id, session)
scan_ctrl.flush(scan, session)
except Exception as e:
log.exception(e)
return
@scan_app.task(ignore_result=True, acks_late=True)
def files_flush(files, scan_id):
try:
with session_transaction() as session:
user = User.get_by_rmqvhost(session)
scan = Scan.get_scan(scan_id, user.id, session)
ftpuser = scan.user.ftpuser
log.debug("Flushing files %s", files)
ftp_ctrl.flush(ftpuser, files)
except Exception as e:
log.exception(e)
return
########################
# command line launcher
########################
if __name__ == '__main__':
options = config.get_celery_options("brain.scan_tasks",
"scan_app")
scan_app.worker_main(options)
tmpname = None
# retrieve queue name and the associated plugin
routing_key = current_task.request.delivery_info['routing_key']
probe = probes[routing_key]
log.debug("filename %s probe %s", filename, probe)
(fd, tmpname) = tempfile.mkstemp()
os.close(fd)
ftp_ctrl.download_file(frontend, filename, tmpname)
results = probe.run(tmpname)
handle_output_files(results, frontend, filename)
return bytes_to_utf8(results)
except Exception as e:
log.exception(e)
raise probe_scan.retry(countdown=2, max_retries=3, exc=e)
finally:
# Some AV always delete suspicious file
if tmpname is not None and os.path.exists(tmpname):
log.debug("filename %s probe %s removing tmp_name %s",
filename, probe, tmpname)
os.remove(tmpname)
########################
# command line launcher
########################
if __name__ == '__main__':
options = config.get_celery_options("probe.tasks",
"probe_app")
probe_app.worker_main(options)
try:
tmpname = None
# retrieve queue name and the associated plugin
probe = probes[routing_key]
log.debug("filename %s probe %s", filename, probe)
(fd, tmpname) = tempfile.mkstemp()
os.close(fd)
ftp_ctrl.download_file(frontend, filename, tmpname)
results = probe.run(tmpname)
handle_output_files(results, frontend, filename)
return bytes_to_utf8(results)
except Exception as e:
log.exception(e)
raise probe_scan.retry(countdown=2, max_retries=3, exc=e)
finally:
# Some AV always delete suspicious file
if tmpname is not None and os.path.exists(tmpname):
log.debug("filename %s probe %s removing tmp_name %s",
filename, probe, tmpname)
os.remove(tmpname)
########################
# command line launcher
########################
if __name__ == '__main__':
options = config.get_celery_options("probe.tasks",
"probe_app")
probe_app.worker_main(options)
def job_success(result, file, probe):
try:
log.info("file:%s probe %s", file, probe)
celery_frontend.scan_result(file, probe, result)
except Exception as e:
log.exception(e)
raise job_success.retry(countdown=5, max_retries=3, exc=e)
@results_app.task(ignore_result=True, acks_late=True)
def job_error(parent_taskid, file, probe):
try:
log.info("file:%s probe %s", file, probe)
with session_query() as session:
result = probe_ctrl.create_error_results(probe, "job error",
session)
celery_frontend.scan_result(file, probe, result)
except Exception as e:
log.exception(e)
raise job_error.retry(countdown=5, max_retries=3, exc=e)
########################
# command line launcher
########################
if __name__ == '__main__':
options = config.get_celery_options("brain.results_tasks",
"brain_results_app")
results_app.worker_main(options)
return nb_files
except (IrmaDatabaseError, IrmaFileSystemError) as e:
log.exception(e)
@frontend_app.task()
def clean_fs_size():
try:
cron_cfg = config.frontend_config['cron_clean_file_size']
max_size = cron_cfg['clean_fs_max_size']
# 0 means disabled
if max_size == '0':
log.debug("disabled by config")
return 0
max_size_bytes = humanfriendly.parse_size(max_size, binary=True)
nb_files = file_ctrl.remove_files_size(max_size_bytes)
log.info("removed %d files", nb_files)
return nb_files
except (IrmaDatabaseError, IrmaFileSystemError) as e:
log.exception(e)
########################
# command line launcher
########################
if __name__ == '__main__': # pragma: no cover
options = config.get_celery_options("api.tasks.frontend_app",
"frontend_app")
frontend_app.worker_main(options)
with session_transaction() as session:
user = User.get_by_rmqvhost(session)
scan = Scan.get_scan(scan_id, user.id, session)
scan_ctrl.flush(scan, session)
except Exception as e:
log.exception(type(e).__name__ + " : " + str(e))
return
@scan_app.task(ignore_result=True, acks_late=True)
def files_flush(files, scan_id):
try:
with session_transaction() as session:
user = User.get_by_rmqvhost(session)
scan = Scan.get_scan(scan_id, user.id, session)
ftpuser = scan.user.ftpuser
log.debug("Flushing files %s", files)
ftp_ctrl.flush(ftpuser, files)
except Exception as e:
log.exception(type(e).__name__ + " : " + str(e))
return
########################
# command line launcher
########################
if __name__ == '__main__':
options = config.get_celery_options("brain.scan_tasks", "scan_app")
scan_app.worker_main(options)