def _new_file(fileobj, session):
sha256 = sha256sum(fileobj)
# split files between subdirs
path = build_sha256_path(sha256)
try:
# The file exists
log.debug("try opening file with sha256: %s", sha256)
file = File.load_from_sha256(sha256, session)
if file.path is None:
log.debug("file sample missing writing it")
save_to_file(fileobj, path)
file.path = path
except IrmaDatabaseResultNotFound:
# It doesn't
time = compat.timestamp()
sha1 = sha1sum(fileobj)
md5 = md5sum(fileobj)
# determine file mimetype
magic = Magic()
# magic only deal with buffer
# feed it with a 4MB buffer
mimetype = magic.from_buffer(fileobj.read(2**22))
size = save_to_file(fileobj, path)
log.debug(
"not present, saving, sha256 %s sha1 %s"
"md5 %s size %s mimetype: %s", sha256, sha1, md5, size, mimetype)
file = File(sha256, sha1, md5, size, mimetype, path, time, time)
session.add(file)
return file
def __init__(self, filename, probename, scanid, taskid):
self.filename = filename
self.probename = probename
self.ts_start = timestamp()
self.status = self.running
self.scan_id = scanid
self.task_id = taskid
def set_result(scanid, file_hash, probe, result):
with session_transaction() as session:
scan = Scan.load_from_ext_id(scanid, session=session)
fws = scan.get_filewebs_by_sha256(file_hash)
if len(fws) == 0:
log.error("file %s not found in scan", file_hash)
return
fws_file = File.load_from_sha256(file_hash, session)
fws_file.timestamp_last_scan = compat.timestamp()
fws_file.update(['timestamp_last_scan'], session=session)
sanitized_res = _sanitize_res(result)
# update results for all files with same sha256
for fw in fws:
# Update main reference results with fresh results
pr = _fetch_probe_result(fw, probe)
_update_ref_results(fw, fw.file, pr)
fw.file.update(session=session)
# fill ProbeResult with probe raw results
pr.doc = sanitized_res
pr.status = sanitized_res.get('status', None)
s_type = sanitized_res.get('type', None)
pr.type = IrmaProbeType.normalize(s_type)
pr.update(session=session)
probedone = []
for fw_pr in fw.probe_results:
if fw_pr.doc is not None:
probedone.append(fw_pr.name)
log.info("scanid: %s result from %s probedone %s", scanid, probe,
probedone)
is_finished(scanid)
def _finish(job_id, status):
with session_transaction() as session:
job = Job.load(job_id, session)
job.status = status
job.ts_end = timestamp()
job.update(['status', 'ts_end'], session)
scan_id = job.scan.id
scanctrl.check_finished(scan_id)
def set_result(self, probe, result):
# Update file last_scan timestamp
self.file.timestamp_last_scan = timestamp()
# Retrieve empty probe result
pr = self.fetch_probe_result(probe)
# Update main reference results with fresh results
pr.file = self.file
# fill ProbeResult with probe raw results
pr.doc = result
pr.status = result.get('status', None)
s_type = result.get('type', None)
pr.type = IrmaProbeType.normalize(s_type)
def new(request):
""" Create a new scan.
The request should be performed using a POST request method.
"""
session = db.session
ip = request.remote_addr
scan = Scan(compat.timestamp(), ip)
session.add(scan)
scan.set_status(IrmaScanStatus.empty)
session.commit()
log.debug("scan %s: created", scan.external_id)
return scan_schema.dump(scan).data
def remove_old_files(cls, max_age, session):
"""Remove the files that are older than timestamp() - max_age
from the file system
:param max_age: the files older than timestamp() - max_age
will be deleted
:rtype: int
:return: the number of deleted files
"""
fl = session.query(cls).filter(
cls.timestamp_last_scan < compat.timestamp() - max_age).filter(
cls.path.isnot(None)).all()
for f in fl:
f.remove_file_from_fs()
return len(fl)
def remaining_quota(self, session):
if self.quota == 0:
# quota=0 means quota disabled
remaining = None
else:
# quota are per 24h
min_ts = timestamp() - 24 * 60 * 60
scan_list = session.query(Scan).filter(Scan.user_id == self.id).filter(Scan.timestamp >= min_ts).all()
consumed = 0
for scan in scan_list:
consumed += scan.nb_jobs
# Quota are set per 24 hours
remaining = self.quota - consumed
return remaining
def remove_old_files(cls, max_age, session):
"""Remove the files that are older than timestamp() - max_age
from the file system
:param max_age: the files older than timestamp() - max_age
will be deleted
:rtype: int
:return: the number of deleted files
"""
fl = session.query(cls).filter(
cls.timestamp_last_scan < compat.timestamp() - max_age
).filter(
cls.path is not None
).all()
for f in fl:
f.remove_file_from_fs()
return len(fl)
def new(db):
""" Create a new scan.
The request should be performed using a POST request method.
"""
try:
ip = request.remote_addr
scan = Scan(compat.timestamp(), ip)
db.add(scan)
scan.set_status(IrmaScanStatus.empty)
db.commit()
log.debug("scanid: %s", scan.external_id)
response.content_type = "application/json; charset=UTF-8"
return scan_schema.dumps(scan).data
except Exception as e:
log.exception(e)
process_error(e)
def new(db):
""" Create a new scan.
The request should be performed using a POST request method.
"""
try:
ip = request.remote_addr
scan = Scan(compat.timestamp(), ip)
db.add(scan)
scan.set_status(IrmaScanStatus.empty)
db.commit()
log.debug("scanid: %s", scan.external_id)
response.content_type = "application/json; charset=UTF-8"
return scan_schema.dumps(scan).data
except Exception as e:
log.exception(e)
process_error(e)
def __init__(self, status, scan):
super(ScanEvents, self).__init__()
self.status = status
self.timestamp = compat.timestamp()
self.scan = scan
def __init__(self, frontend_scanid, user_id):
self.scan_id = frontend_scanid
self.status = IrmaScanStatus.empty
self.timestamp = timestamp()
self.user_id = user_id
def launch_v2(request, body):
""" Launch a scan.
The request should be performed using a POST request method.
The input json format is the following:
{
files: [fileext1, fileext2...]
options:
probes: list of probes or None for all available,
force: boolean (default False),
mimetype_filtering: boolean (default True),
resubmit_files: boolean (default True),
}
"""
scan_params = body
if not scan_params:
raise HTTPInvalidParam("Missing json parameters", "body")
files_list = body.get('files', None)
if files_list is None or len(files_list) == 0:
raise HTTPInvalidParam("Missing values", "files")
# Set default values
force = True
mimetype_filtering = True
resubmit_files = True
probes = None
# override with given values if set
scan_options = body.get("options", None)
if scan_options is not None:
force = scan_options.get("force", False)
if type(force) is not bool:
raise HTTPInvalidParam("Should be boolean", "force")
mimetype_filtering = scan_options.get("mimetype_filtering", True)
if type(mimetype_filtering) is not bool:
raise HTTPInvalidParam("Should be boolean", "mimetype_filtering")
resubmit_files = scan_options.get("resubmit_files", True)
if type(resubmit_files) is not bool:
raise HTTPInvalidParam("Should be boolean", "resubmit_files")
probes = scan_options.get("probes", None)
session = db.session
ip = request.remote_addr
scan = Scan(compat.timestamp(), ip)
session.add(scan)
# handle scan parameter
# cached results: "force" (default: True)
scan.force = force
# use mimetype for probelist: "mimetype_filtering" (default: True)
scan.mimetype_filtering = mimetype_filtering
# rescan file outputted from probes "resubmit_files" (default: True)
scan.resubmit_files = resubmit_files
scan.set_status(IrmaScanStatus.empty)
session.commit()
log.debug("scan %s: created", scan.external_id)
msg = "scan %s: Force %s MimeF %s"
msg += " Resub %s Probes %s"
log.debug(msg, scan.external_id, scan.force, scan.mimetype_filtering,
scan.resubmit_files, probes)
for fe_id in files_list:
log.info("scan %s adding file %s", scan.external_id, fe_id)
try:
file_ext = FileExt.load_from_ext_id(fe_id, session)
except IrmaDatabaseResultNotFound:
raise HTTPInvalidParam("File %s not found" % fe_id, "files")
if file_ext.file.path is None:
raise HTTPInvalidParam(
"File with hash %s should be ("
"re)uploaded" % file_ext.file.sha256, "files")
if file_ext.scan is not None:
raise HTTPInvalidParam("File %s already scanned" % fe_id, "files")
file_ext.scan = scan
scan.set_status(IrmaScanStatus.ready)
session.commit()
probelist = probe_ctrl.check_probe(probes)
scan.set_probelist(probelist)
session.commit()
# launch_asynchronous scan via frontend task
celery_frontend.scan_launch(str(scan.external_id))
return scan_schema.dump(scan).data
def __init__(self, frontend_scanid, user_id, nb_files):
self.scan_id = frontend_scanid
self.status = IrmaScanStatus.empty
self.timestamp = timestamp()
self.nb_files = nb_files
self.user_id = user_id
def __init__(self, status, scan):
super(ScanEvents, self).__init__()
self.status = status
self.timestamp = compat.timestamp()
self.scan = scan