def new(db):
""" Add a new tag, text is passed as query parameter.
:return:
on success new tag object is returned
on error 'msg' gives reason message
"""
try:
# Check if parameter is set
if 'text' not in request.json:
raise ValueError("Missing text parameter")
text = request.json['text']
# Check if a tag with same tag already exists
available_tags = Tag.query_find_all(db)
if text in [t.text for t in available_tags]:
raise ValueError("Tag already exists")
# create tag if not already present
tag = Tag(text)
db.add(tag)
db.commit()
# returns new tag
response.content_type = "application/json; charset=UTF-8"
return tag_schema.dumps(tag).data
except Exception as e:
process_error(e)
def list(db):
""" Get a list of all scans which have been launched.
"""
try:
# Get values from query or set to default
offset = request.query.get("offset", default=0)
offset = int(offset)
limit = request.query.get("limit", default=5)
limit = int(limit)
log.debug("offset %s limit %s", offset, limit)
base_query = Scan.query_joined(db)
items = base_query.limit(limit).offset(offset).all()
if offset == 0 and len(items) < limit:
total = len(items)
else:
total = base_query.count()
log.debug("found %s scans", total)
response.content_type = "application/json; charset=UTF-8"
return {
"total": total,
"offset": offset,
"limit": limit,
"data": scan_schema.dump(items, many=True).data,
}
except Exception as e:
log.exception(e)
process_error(e)
def add_files(scanid, db):
""" Attach a file to a scan.
The request should be performed using a POST request method.
"""
try:
log.debug("scanid %s", scanid)
validate_id(scanid)
scan = Scan.load_from_ext_id(scanid, db)
if len(request.files) == 0:
raise ValueError("No files uploaded")
files = {}
for f in request.files:
upfile = request.files.get(f)
filename = decode_utf8(upfile.raw_filename)
data = upfile.file
files[filename] = data
scan_ctrl.add_files(scan, files, db)
response.content_type = "application/json; charset=UTF-8"
return scan_schema.dumps(scan).data
except Exception as e:
log.exception(e)
process_error(e)
def new(db):
""" Add a new tag, text is passed as query parameter.
:return:
on success new tag object is returned
on error 'msg' gives reason message
"""
try:
# Check if parameter is set
if 'text' not in request.json:
raise ValueError("Missing text parameter")
text = request.json['text']
# Check if a tag with same tag already exists
available_tags = Tag.query_find_all(db)
if text in [t.text for t in available_tags]:
raise ValueError("Tag already exists")
# create tag if not already present
tag = Tag(text)
db.add(tag)
db.commit()
# returns new tag
response.content_type = "application/json; charset=UTF-8"
return tag_schema.dumps(tag).data
except Exception as e:
process_error(e)
def get(sha256, db):
""" Detail about one file and all known scans summary where file was
present (identified by sha256). Support pagination.
:param all params are sent using query method
:param if alt parameter is "media", response will contains the binary data
:rtype: dict of 'total': int, 'page': int, 'per_page': int,
:return:
on success fileinfo contains file information
on success 'items' contains a list of files found
on error 'msg' gives reason message
"""
try:
log.debug("h_value %s", sha256)
# Check wether its a download attempt or not
if request.query.alt == "media":
return _download(sha256, db)
# Get values from query or default
offset = request.query.get("offset", default=0)
offset = int(offset)
limit = request.query.get("limit", default=25)
limit = int(limit)
file = File.load_from_sha256(sha256, db)
# query all known results not only those with different names
base_query = FileWeb.query_find_by_hash("sha256",
sha256,
None,
db,
distinct_name=False)
# TODO: Find a way to move pagination as a BaseQuery like in
# flask_sqlalchemy.
# https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422
items = base_query.limit(limit).offset(offset).all()
if offset == 0 and len(items) < limit:
total = len(items)
else:
total = base_query.count()
log.debug("offset %d limit %d total %d", offset, limit, total)
file_web_schema = FileWebSchema_v1_1(exclude=('probe_results',
'file_infos'))
fileinfo_schema = FileSchema_v1_1()
# TODO: allow formatted to be a parameter
formatted = True
fileinfo_schema.context = {'formatted': formatted}
response.content_type = "application/json; charset=UTF-8"
return {
'file_infos': fileinfo_schema.dump(file).data,
'total': total,
'offset': offset,
'limit': limit,
'items': file_web_schema.dump(items, many=True).data,
}
except Exception as e:
log.exception(e)
process_error(e)
def test004_process_database_error(self, m_sys, m_abort):
m_sys.exc_info.return_value = (MagicMock(), None, MagicMock())
error = IrmaDatabaseError("message_error")
exp_code = 402
exp_error = api_errors.ApiError("database_error")
api_errors.process_error(error)
m_abort.assert_called_once()
call_args = m_abort.call_args[0]
self.assertEqual(call_args[0], exp_code)
self.assertEqual(str(call_args[1]), str(exp_error))
def test003_process_notfound_error(self, m_sys, m_abort):
m_sys.exc_info.return_value = (MagicMock(), None, MagicMock())
error = IrmaDatabaseResultNotFound("message_error")
exp_code = 404
exp_error = api_errors.ApiError("request_error", "Object not Found")
api_errors.process_error(error)
m_abort.assert_called_once()
call_args = m_abort.call_args[0]
self.assertEqual(call_args[0], exp_code)
self.assertEqual(str(call_args[1]), str(exp_error))
def test004_process_database_error(self, m_sys, m_abort):
m_sys.exc_info.return_value = (MagicMock(), None, MagicMock())
error = IrmaDatabaseError("message_error")
exp_code = 402
exp_error = api_errors.ApiError("database_error")
api_errors.process_error(error)
m_abort.assert_called_once()
call_args = m_abort.call_args[0]
self.assertEqual(call_args[0], exp_code)
self.assertEqual(str(call_args[1]), str(exp_error))
def test003_process_notfound_error(self, m_sys, m_abort):
m_sys.exc_info.return_value = (MagicMock(), None, MagicMock())
error = IrmaDatabaseResultNotFound("message_error")
exp_code = 404
exp_error = api_errors.ApiError("request_error", "Object not Found")
api_errors.process_error(error)
m_abort.assert_called_once()
call_args = m_abort.call_args[0]
self.assertEqual(call_args[0], exp_code)
self.assertEqual(str(call_args[1]), str(exp_error))
def remove_tag(sha256, tagid, db):
""" Remove a tag attached to a file.
"""
try:
log.debug("h_value %s tagid %s", sha256, tagid)
fobj = File.load_from_sha256(sha256, db)
fobj.remove_tag(tagid, db)
db.commit()
except Exception as e:
log.exception(e)
process_error(e)
def add_tag(sha256, tagid, db):
""" Attach a tag to a file.
"""
try:
log.debug("h_value %s tagid %s", sha256, tagid)
fobj = File.load_from_sha256(sha256, db)
fobj.add_tag(tagid, db)
db.commit()
except Exception as e:
log.exception(e)
process_error(e)
def list():
""" get active probe list. This list is used to launch a scan.
"""
try:
probelist = celery_brain.probe_list()
log.debug("probe list: %s", "-".join(probelist))
response.content_type = "application/json; charset=UTF-8"
return {"total": len(probelist), "data": probelist}
except Exception as e:
log.exception(e)
process_error(e)
def get(scanid, db):
""" Retrieve information for a specific scan
"""
try:
log.debug("scanid %s", scanid)
validate_id(scanid)
scan = Scan.load_from_ext_id(scanid, db)
response.content_type = "application/json; charset=UTF-8"
return scan_schema.dumps(scan).data
except Exception as e:
log.exception(e)
process_error(e)
def list():
""" get active probe list. This list is used to launch a scan.
"""
try:
probelist = celery_brain.probe_list()
log.debug("probe list: %s", "-".join(probelist))
response.content_type = "application/json; charset=UTF-8"
return {
"total": len(probelist),
"data": probelist
}
except Exception as e:
log.exception(e)
process_error(e)
def list(db):
""" Search for all tags in TAG table using query.
:return:
on success 'items' contains a list of all tags
on error 'msg' gives reason message
"""
try:
available_tags = Tag.query_find_all(db)
response.content_type = "application/json; charset=UTF-8"
return {
'items': tag_schema.dump(available_tags, many=True).data,
}
except Exception as e:
process_error(e)
def list(db):
""" Search for all tags in TAG table using query.
:return:
on success 'items' contains a list of all tags
on error 'msg' gives reason message
"""
try:
available_tags = Tag.query_find_all(db)
response.content_type = "application/json; charset=UTF-8"
return {
'items': tag_schema.dump(available_tags, many=True).data,
}
except Exception as e:
process_error(e)
def cancel(scanid, db):
""" Cancel a scan.
The request should be performed using a POST request method.
"""
try:
log.debug("scanid %s", scanid)
validate_id(scanid)
scan = Scan.load_from_ext_id(scanid, db)
scan_ctrl.cancel(scan, db)
response.content_type = "application/json; charset=UTF-8"
return scan_schema.dumps(scan).data
except Exception as e:
log.exception(e)
process_error(e)
def get_results(scanid, db):
""" Retrieve results for a scan. Results are the same as in the get()
method, i.e. a summary for each scanned files.
The request should be performed using a GET request method.
"""
try:
log.debug("scanid %s", scanid)
validate_id(scanid)
scan = Scan.load_from_ext_id(scanid, db)
file_web_schema = FileWebSchema_v1_1(exclude=('probe_results',
'file_infos'))
response.content_type = "application/json; charset=UTF-8"
return file_web_schema.dumps(scan.files_web, many=True).data
except Exception as e:
log.exception(e)
process_error(e)
def new(db):
""" Create a new scan.
The request should be performed using a POST request method.
"""
try:
ip = request.remote_addr
scan = Scan(compat.timestamp(), ip)
db.add(scan)
scan.set_status(IrmaScanStatus.empty)
db.commit()
log.debug("scanid %s", scan.external_id)
response.content_type = "application/json; charset=UTF-8"
return scan_schema.dumps(scan).data
except Exception as e:
log.exception(e)
process_error(e)
def launch(scanid, db):
""" Launch a scan.
The request should be performed using a POST request method.
"""
try:
validate_id(scanid)
scan = Scan.load_from_ext_id(scanid, db)
probes = None
# handle scan parameter
# cached results: "force" (default: True)
scan.force = False
if 'force' in request.json and request.json.get('force') is True:
scan.force = True
# use mimetype for probelist: "mimetype_filtering" (default: True)
scan.mimetype_filtering = True
if 'mimetype_filtering' in request.json and \
request.json.get('mimetype_filtering') is False:
scan.mimetype_filtering = False
# rescan file outputted from probes "resubmit_files" (default: True)
scan.resubmit_files = True
if 'resubmit_files' in request.json and \
request.json.get('resubmit_files') is False:
scan.resubmit_files = False
db.commit()
# handle scan parameter / probelist: "probes"
if 'probes' in request.json:
probes = request.json.get('probes').split(',')
msg = "scanid %s Force %s MimeF %s"
msg += "Resub %s Probes %s"
log.debug(msg, scanid, scan.force, scan.mimetype_filtering,
scan.resubmit_files, probes)
scan_ctrl.check_probe(scan, probes, db)
# launch_asynchronous scan via frontend task
celery_frontend.scan_launch(scanid)
response.content_type = "application/json; charset=UTF-8"
return scan_schema.dumps(scan).data
except Exception as e:
log.exception(e)
process_error(e)
def get(resultid, db):
""" Retrieve a single fileweb result, with details.
The request should be performed using a GET request method.
"""
try:
formatted = False if request.query.formatted == 'no' else True
log.debug("resultid %s formatted %s", resultid, formatted)
validate_id(resultid)
fw = FileWeb.load_from_ext_id(resultid, db)
file_web_schema = FileWebSchema_v1_1()
file_web_schema.context = {'formatted': formatted}
response.content_type = "application/json; charset=UTF-8"
return file_web_schema.dumps(fw).data
except Exception as e:
log.exception(e)
process_error(e)
def get_stats(db):
""" Search a file using query filters (tags + hash or name). Support
pagination.
:param all params are sent using query method
:rtype: dict of 'total': int, 'page': int, 'per_page': int,
'items': list of file(s) found
:return:
on success 'items' contains a list of files found
on error 'msg' gives reason message
"""
try:
name = None
if 'name' in request.query:
name = decode_utf8(request.query['name'])
h_value = request.query.hash or None
search_tags = request.query.tags or None
if search_tags is not None:
search_tags = search_tags.split(',')
log.debug("name %s h_value %s search_tags %s",
name, h_value, search_tags)
if name is not None and h_value is not None:
raise ValueError("Can't find using both name and hash")
# Options query
offset = int(request.query.offset) if request.query.offset else 0
limit = int(request.query.limit) if request.query.limit else 0
if name is not None:
base_query = FileWeb.query_find_by_name(name, search_tags, db)
elif h_value is not None:
h_type = guess_hash_type(h_value)
if h_type is None:
raise ValueError("Hash not supported")
base_query = FileWeb.query_find_by_hash(
h_type, h_value, search_tags, db)
else:
# FIXME this is just a temporary way to output
# all files, need a dedicated
# file route and controller
base_query = FileWeb.query_find_by_name("", search_tags, db)
# TODO: Find a way to move pagination as a BaseQuery like in
# flask_sqlalchemy.
# https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422
#items = base_query.limit(limit).offset(offset).all()
items = base_query.all()
files_infos = file_web_schema.dump(items, many=True).data
# Process items found statistiques.
#log.debug("Debug ::::::::::::: items : %s", type(items))
#log.debug("Debug ::::::::::::: files_infos: %s", type(files_infos))
stats = []
stats_fake = [
{'name': 'armaditoAV', 'version':3.14, 'nbsamples':2, 'malware': 1, 'clean':1, 'errors':0},
{'name': 'clamav', 'version':3.14, 'nbsamples':2, 'malware': 1, 'clean':1, 'errors':0},
]
for i, val in enumerate(files_infos):
#dsds
log.debug("Debug :::::::::::::: results : %s :: %s", type(val['probe_results']), val['probe_results'])
#log.debug("Debug :::::::::::::: results : %s", type(val.probe_results))
probe_results = val['probe_results']
for j, res in enumerate(probe_results):
#log.debug("Debug :::::::::::::: probe_result : %s", type(res))
# Get av name
#log.debug("Debug :::::::::::::: av_name : %s", res.name)
#log.debug("Debug :::::::::::::: av_type : %s", res.type)
#log.debug("Debug :::::::::::::: av_version : %s", res.version)
if res.type == "antivirus":
add_stats(stats,res)
if offset == 0 and len(items) < limit:
total = len(items)
else:
total = base_query.count()
log.debug("Found %s results", total)
response.content_type = "application/json; charset=UTF-8"
return {
'total': total,
'offset': offset,
'limit': limit,
'items': stats,
}
except Exception as e:
log.exception(e)
process_error(e)
def list(db):
""" Search a file using query filters (tags + hash or name). Support
pagination.
:param all params are sent using query method
:rtype: dict of 'total': int, 'page': int, 'per_page': int,
'items': list of file(s) found
:return:
on success 'items' contains a list of files found
on error 'msg' gives reason message
"""
try:
name = None
if 'name' in request.query:
name = decode_utf8(request.query['name'])
h_value = request.query.hash or None
search_tags = request.query.tags or None
if search_tags is not None:
search_tags = search_tags.split(',')
log.debug("name %s h_value %s search_tags %s", name, h_value,
search_tags)
if name is not None and h_value is not None:
raise ValueError("Can't find using both name and hash")
# Options query
offset = int(request.query.offset) if request.query.offset else 0
limit = int(request.query.limit) if request.query.limit else 25
if name is not None:
base_query = FileWeb.query_find_by_name(name, search_tags, db)
elif h_value is not None:
h_type = guess_hash_type(h_value)
if h_type is None:
raise ValueError("Hash not supported")
base_query = FileWeb.query_find_by_hash(h_type, h_value,
search_tags, db)
else:
# FIXME this is just a temporary way to output
# all files, need a dedicated
# file route and controller
base_query = FileWeb.query_find_by_name("", search_tags, db)
# TODO: Find a way to move pagination as a BaseQuery like in
# flask_sqlalchemy.
# https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422
items = base_query.limit(limit).offset(offset).all()
if offset == 0 and len(items) < limit:
total = len(items)
else:
total = base_query.count()
log.debug("Found %s results", total)
response.content_type = "application/json; charset=UTF-8"
return {
'total': total,
'offset': offset,
'limit': limit,
'items': file_web_schema.dump(items, many=True).data,
}
except Exception as e:
log.exception(e)
process_error(e)
def get_archive(db):
""" Search a file using query filters (tags + hash or name). Support
pagination.
:param all params are sent using query method
:rtype: dict of 'total': int, 'page': int, 'per_page': int,
'items': list of file(s) found
:return:
on success 'items' contains a list of files found
on error 'msg' gives reason message
"""
try:
name = None
if 'name' in request.query:
name = decode_utf8(request.query['name'])
h_value = request.query.hash or None
search_tags = request.query.tags or None
if search_tags is not None:
search_tags = search_tags.split(',')
log.debug("name %s h_value %s search_tags %s",
name, h_value, search_tags)
if name is not None and h_value is not None:
raise ValueError("Can't find using both name and hash")
# Options query
offset = int(request.query.offset) if request.query.offset else 0
limit = int(request.query.limit) if request.query.limit else 25
if name is not None:
base_query = FileWeb.query_find_by_name(name, search_tags, db)
elif h_value is not None:
h_type = guess_hash_type(h_value)
if h_type is None:
raise ValueError("Hash not supported")
base_query = FileWeb.query_find_by_hash(
h_type, h_value, search_tags, db)
else:
# FIXME this is just a temporary way to output
# all files, need a dedicated
# file route and controller
base_query = FileWeb.query_find_by_name("", search_tags, db)
# TODO: Find a way to move pagination as a BaseQuery like in
# flask_sqlalchemy.
# https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422
items = base_query.limit(limit).offset(offset).all()
# get file basic information
file_web_schema = FileWebSchema_v1_1(exclude=('probe_results','file_infos'))
infos = file_web_schema.dump(items, many=True).data
sha256_list = []
for i, val in enumerate(items):
#log.debug("Debug :: items[%s] = %s ::",i, val.file.sha256)
fhash = val.file.sha256
sha256_list.append(fhash)
#file_web.file.sha256
if sha256_list is not None:
return _download_zip(sha256_list,db,infos)
except Exception as e:
log.exception(e)
process_error(e)
