def set_result(self, probe, result):
# Update file last_scan timestamp
self.file.timestamp_last_scan = timestamp()
# Retrieve empty probe result
pr = self.fetch_probe_result(probe)
# Update main reference results with fresh results
pr.file = self.file
# fill ProbeResult with probe raw results
pr.doc = result
pr.status = result.get('status', None)
s_type = result.get('type', None)
pr.type = IrmaProbeType.normalize(s_type)
def new(request):
""" Create a new scan.
The request should be performed using a POST request method.
"""
session = db.session
ip = request.remote_addr
scan = Scan(compat.timestamp(), ip)
session.add(scan)
scan.set_status(IrmaScanStatus.empty)
session.commit()
log.debug("scan %s: created", scan.external_id)
return scan_schema.dump(scan).data
def new(request):
""" Create a new scan.
The request should be performed using a POST request method.
"""
session = db.session
ip = request.remote_addr
scan = Scan(compat.timestamp(), ip)
session.add(scan)
scan.set_status(IrmaScanStatus.empty)
session.commit()
log.debug("scan %s: created", scan.external_id)
return scan_schema.dump(scan).data
def remove_old_files(cls, max_age, session):
"""Remove the files that are older than timestamp() - max_age
from the file system
:param max_age: the files older than timestamp() - max_age
will be deleted
:param session: SQLA session
:rtype: int
:return: the number of deleted files
"""
fl = session.query(cls).filter(
cls.timestamp_last_scan < compat.timestamp() - max_age).filter(
cls.path.isnot(None)).all()
for f in fl:
f.remove_file_from_fs()
session.commit()
return len(fl)
def remove_old_files(cls, max_age, session):
"""Remove the files that are older than timestamp() - max_age
from the file system
:param max_age: the files older than timestamp() - max_age
will be deleted
:param session: SQLA session
:rtype: int
:return: the number of deleted files
"""
fl = session.query(cls).filter(
cls.timestamp_last_scan < compat.timestamp() - max_age
).filter(
cls.path.isnot(None)
).all()
for f in fl:
f.remove_file_from_fs()
session.commit()
return len(fl)
def quick_scan(request):
""" Launch a scan for one file
"""
session = db.session
ip = request.remote_addr
# Create file
fe = files_ctrl.create(request)
fe = FileExt.load_from_ext_id(fe['result_id'], session)
# Create a scan
scan = Scan(compat.timestamp(), ip, files_ext=[fe, ])
session.add(scan)
session.commit()
# launch_asynchronous scan via frontend task
celery_frontend.scan_launch(str(scan.external_id))
return scan_schema.dump(scan).data
def quick_scan(request):
""" Launch a scan for one file
"""
session = db.session
ip = request.remote_addr
# Create file
fe = files_ctrl.create(request)
fe = FileExt.load_from_ext_id(fe['result_id'], session)
# Create a scan
scan = Scan(compat.timestamp(), ip, files_ext=[fe, ])
session.add(scan)
session.commit()
# launch_asynchronous scan via frontend task
celery_frontend.scan_launch(str(scan.external_id))
return scan_schema.dump(scan).data
def create(cls, fileobj, sha256, path, session):
try:
# Create a new file
time = compat.timestamp()
sha1 = sha1sum(fileobj)
md5 = md5sum(fileobj)
# determine file mimetype
magic = Magic()
# magic only deal with buffer
# feed it with a 4MB buffer
mimetype = magic.from_buffer(fileobj.read(2 ** 22))
size = save_to_file(fileobj, path)
log.debug("not present, saving, sha256 %s sha1 %s"
"md5 %s size %s mimetype: %s",
sha256, sha1, md5, size, mimetype)
file = File(sha256, sha1, md5, size, mimetype, path, time, time)
session.add(file)
session.commit()
return file
except IntegrityError:
raise IrmaDatabaseError("Integrity error")
def create(cls, fileobj, sha256, path, session):
try:
# Create a new file
time = compat.timestamp()
sha1 = sha1sum(fileobj)
md5 = md5sum(fileobj)
# determine file mimetype
magic = Magic()
# magic only deal with buffer
# feed it with a 4MB buffer
mimetype = magic.from_buffer(fileobj.read(2**22))
size = save_to_file(fileobj, path)
log.debug(
"not present, saving, sha256 %s sha1 %s"
"md5 %s size %s mimetype: %s", sha256, sha1, md5, size,
mimetype)
file = File(sha256, sha1, md5, size, mimetype, path, time, time)
session.add(file)
session.commit()
return file
except IntegrityError as e:
try:
session.rollback()
file = File.load_from_sha256(sha256, session)
log.debug("Integrity error but load "
"successful for file.%s", sha256)
return file
except NoResultFound:
log.debug("Integrity error load failed for file.%s", sha256)
pass
log.error(
"Database integrity error: refuse to insert file.%s. %s",
sha256,
e,
)
raise IrmaDatabaseError("Integrity error")
def launch_v2(request, body):
""" Launch a scan.
The request should be performed using a POST request method.
The input json format is the following:
{
files: [fileext1, fileext2...]
options:
probes: list of probes or None for all available,
force: boolean (default False),
mimetype_filtering: boolean (default True),
resubmit_files: boolean (default True),
}
"""
scan_params = body
if not scan_params:
raise HTTPInvalidParam("Missing json parameters", "body")
files_list = body.get('files', None)
if files_list is None or len(files_list) == 0:
raise HTTPInvalidParam("Missing values", "files")
# Set default values
force = True
mimetype_filtering = True
resubmit_files = True
probes = None
# override with given values if set
scan_options = body.get("options", None)
if scan_options is not None:
force = scan_options.get("force", force)
if type(force) is not bool:
raise HTTPInvalidParam("Should be boolean", "force")
mimetype_filtering = scan_options.get("mimetype_filtering",
mimetype_filtering)
if type(mimetype_filtering) is not bool:
raise HTTPInvalidParam("Should be boolean",
"mimetype_filtering")
resubmit_files = scan_options.get("resubmit_files", resubmit_files)
if type(resubmit_files) is not bool:
raise HTTPInvalidParam("Should be boolean",
"resubmit_files")
probes = scan_options.get("probes", probes)
session = db.session
ip = request.remote_addr
files_ext = []
for fe_id in files_list:
try:
file_ext = FileExt.load_from_ext_id(fe_id, session)
except IrmaDatabaseResultNotFound:
raise HTTPInvalidParam("File %s not found" % fe_id,
"files")
if file_ext.file.path is None:
raise HTTPInvalidParam("File with hash %s should be ("
"re)uploaded" %
file_ext.file.sha256,
"files")
if file_ext.scan is not None:
raise HTTPInvalidParam("File %s already scanned" %
fe_id,
"files")
files_ext.append(file_ext)
scan = Scan(compat.timestamp(), ip,
force=force, mimetype_filtering=mimetype_filtering,
resubmit_files=resubmit_files, files_ext=files_ext,
probes=probes)
session.add(scan)
session.commit()
# launch_asynchronous scan via frontend task
celery_frontend.scan_launch(str(scan.external_id))
return scan_schema.dump(scan).data
def launch_v2(request, body):
""" Launch a scan.
The request should be performed using a POST request method.
The input json format is the following:
{
files: [fileext1, fileext2...]
options:
probes: list of probes or None for all available,
force: boolean (default False),
mimetype_filtering: boolean (default True),
resubmit_files: boolean (default True),
}
"""
scan_params = body
if not scan_params:
raise HTTPInvalidParam("Missing json parameters", "body")
files_list = body.get('files', None)
if files_list is None or len(files_list) == 0:
raise HTTPInvalidParam("Missing values", "files")
# Set default values
force = True
mimetype_filtering = True
resubmit_files = True
probes = None
# override with given values if set
scan_options = body.get("options", None)
if scan_options is not None:
force = scan_options.get("force", force)
if type(force) is not bool:
raise HTTPInvalidParam("Should be boolean", "force")
mimetype_filtering = scan_options.get("mimetype_filtering",
mimetype_filtering)
if type(mimetype_filtering) is not bool:
raise HTTPInvalidParam("Should be boolean",
"mimetype_filtering")
resubmit_files = scan_options.get("resubmit_files", resubmit_files)
if type(resubmit_files) is not bool:
raise HTTPInvalidParam("Should be boolean",
"resubmit_files")
probes = scan_options.get("probes", probes)
session = db.session
ip = request.remote_addr
files_ext = []
for fe_id in files_list:
try:
file_ext = FileExt.load_from_ext_id(fe_id, session)
except IrmaDatabaseResultNotFound:
raise HTTPInvalidParam("File %s not found" % fe_id,
"files")
if file_ext.file.path is None:
raise HTTPInvalidParam("File with hash %s should be ("
"re)uploaded" %
file_ext.file.sha256,
"files")
if file_ext.scan is not None:
raise HTTPInvalidParam("File %s already scanned" %
fe_id,
"files")
files_ext.append(file_ext)
scan = Scan(compat.timestamp(), ip,
force=force, mimetype_filtering=mimetype_filtering,
resubmit_files=resubmit_files, files_ext=files_ext,
probes=probes)
session.add(scan)
session.commit()
# launch_asynchronous scan via frontend task
celery_frontend.scan_launch(str(scan.external_id))
return scan_schema.dump(scan).data
def __init__(self, frontend_scanid, user_id):
self.scan_id = frontend_scanid
self.status = IrmaScanStatus.empty
self.timestamp = timestamp()
self.user_id = user_id
def __init__(self, status, scan):
super(ScanEvents, self).__init__()
self.status = status
self.timestamp = compat.timestamp()
self.scan = scan
def __init__(self, status, scan):
super(ScanEvents, self).__init__()
self.status = status
self.timestamp = compat.timestamp()
self.scan = scan