def create(request):
""" Create a file_ext (could be later attached to a scan
The request should be performed using a POST request method.
Input format is multipart-form-data with file and a json containing
at least the submitter type
"""
log.debug("create file")
session = db.session
# request._params is init by Falcon
# Multipart Middleware giving a dict of part in the form
form_dict = request._params
if 'files' not in form_dict:
raise HTTPInvalidParam("Empty list", "files")
form_file = form_dict.pop('files')
if type(form_file) is list:
raise HTTPInvalidParam("Only one file at a time", "files")
if 'json' not in form_dict:
raise HTTPInvalidParam("Missing json parameter", "json")
payload = json.loads(form_dict['json'])
submitter = payload.pop('submitter')
# ByteIO object is in file
data = form_file.file
filename = decode_utf8(form_file.filename)
file = File.get_or_create(data, session)
file_ext = new_file_ext(submitter, file, filename, payload)
session.add(file_ext)
session.commit()
log.debug("filename: %s file_ext: %s created", filename,
file_ext.external_id)
schema = get_file_ext_schemas(file_ext.submitter)
schema.exclude += ("probe_results", )
return schema.dump(file_ext).data
def add_files(scanid, db):
""" Attach a file to a scan.
The request should be performed using a POST request method.
"""
try:
log.debug("scanid: %s", scanid)
validate_id(scanid)
scan = Scan.load_from_ext_id(scanid, db)
if len(request.files) == 0:
raise ValueError("No files uploaded")
files = {}
for f in request.files:
upfile = request.files.get(f)
filename = decode_utf8(upfile.raw_filename)
data = upfile.file
files[filename] = data
scan_ctrl.add_files(scan, files, db)
response.content_type = "application/json; charset=UTF-8"
return scan_schema.dumps(scan).data
except Exception as e:
log.exception(e)
process_error(e)
def add_files(scanid, db):
""" Attach a file to a scan.
The request should be performed using a POST request method.
"""
try:
log.debug("scanid: %s", scanid)
validate_id(scanid)
scan = Scan.load_from_ext_id(scanid, db)
if len(request.files) == 0:
raise ValueError("No files uploaded")
files = {}
for f in request.files:
upfile = request.files.get(f)
filename = decode_utf8(upfile.raw_filename)
data = upfile.file
files[filename] = data
scan_ctrl.add_files(scan, files, db)
response.content_type = "application/json; charset=UTF-8"
return scan_schema.dumps(scan).data
except Exception as e:
log.exception(e)
process_error(e)
def list(name: text = None,
hash: text = None,
tags: comma_separated_list = None,
offset: number = 0,
limit: number = 25):
""" Search a file using query filters (tags + hash or name). Support
pagination.
:param name: lookup name
:param hash: lookup hash value
:param tags: lookup tag list
:rtype: dict of 'total': int, 'page': int, 'per_page': int,
'items': list of file(s) found
:return:
on success 'items' contains a list of files found
on error 'msg' gives reason message
"""
session = db.session
if name is not None:
name = decode_utf8(name)
log.debug("name %s h_value %s tags %s", name, hash, tags)
if name is not None and hash is not None:
raise ValueError("Can't find using both name and hash")
if name is not None:
base_query = FileExt.query_find_by_name(name, tags, session)
elif hash is not None:
h_type = guess_hash_type(hash)
if h_type is None:
raise ValueError("Hash not supported")
base_query = FileExt.query_find_by_hash(h_type, hash, tags, session)
else:
# FIXME this is just a temporary way to output
# all files, need a dedicated
# file route and controller
base_query = FileExt.query_find_by_name("", tags, session)
# TODO: Find a way to move pagination as a BaseQuery like in
# flask_sqlalchemy.
# https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422
items = base_query.limit(limit).offset(offset).all()
if offset == 0 and len(items) < limit:
total = len(items)
else:
total = base_query.count()
log.debug("Found %s results", total)
return {
'total': total,
'offset': offset,
'limit': limit,
'items': file_ext_schema_lite.dump(items, many=True).data,
}
def add_files(request, scan_id: uuid):
""" Attach a file to a scan.
The request should be performed using a POST request method.
"""
log.debug("scan %s: add_files", scan_id)
session = db.session
scan = Scan.load_from_ext_id(scan_id, session)
IrmaScanStatus.filter_status(scan.status, IrmaScanStatus.empty,
IrmaScanStatus.ready)
# request._params is init by Falcon
form_dict = request._params
if len(form_dict) == 0:
raise HTTPInvalidParam("Empty list", "files")
for (_, f) in form_dict.items():
# Multipart Middleware giving a dict of uploaded files
filename = decode_utf8(f.filename)
# ByteIO object is in file
data = f.file
log.debug("scan %s: add filename: %s", scan_id, filename)
file = File.get_or_create(data, session)
# Legacy v1.1 as we dont received a submitter parameter
# choose one from FileCli/FileWeb based on path value
(path, name) = ntpath.split(filename)
if path != "":
file_ext = FileCli(file, filename)
else:
file_ext = FileWeb(file, filename)
session.add(file_ext)
file_ext.scan = scan
session.commit()
scan.set_status(IrmaScanStatus.ready)
session.commit()
return scan_schema.dump(scan).data
from frontend.models.sqlobjects import Tag
from api.common.sessions import session_transaction
from lib.common.utils import decode_utf8
if len(sys.argv) != 2:
print("usage: {0} <tag_list> (comma separated)".format(sys.argv[0]))
sys.exit(1)
# get tag list as argument
tag_list = sys.argv[1]
# split comma separated list
tags = tag_list.split(",")
# force all tags to lowercase
tags = map(lambda x: decode_utf8(x.lower()), tags)
with session_transaction() as session:
# get all existing tags
existing_tags = Tag.query_find_all(session)
existing_text = [t.text for t in existing_tags]
# filter only the one needed to be created
to_create_tags = filter(lambda x: x not in existing_text, tags)
print u"[+] Tags already existing: {0}".format(",".join(existing_text))
for tag in to_create_tags:
t = Tag(tag)
print u"[+] creating Tag: {0}".format(tag)
session.add(t)
def files(db):
""" Search a file using query filters (hash or name). Support
pagination.
:param all params are send using query method
:rtype: dict of 'total': int, 'page': int, 'per_page': int,
'items': list of file(s) found
:return:
on success 'items' contains a list of files found
on error 'msg' gives reason message
"""
try:
name = None
if 'name' in request.query:
name = decode_utf8(request.query['name'])
h_value = request.query.get('hash')
log.debug("name %s h_value %s", name, h_value)
if name is not None and h_value is not None:
raise ValueError("Can't find using both name and hash")
# Get values from Query or Default
offset = request.query.get("offset", default=0)
offset = int(offset)
limit = request.query.get("limit", default=25)
limit = int(limit)
if name is not None:
base_query = FileWeb.query_find_by_name(name, None, db)
elif h_value is not None:
h_type = guess_hash_type(h_value)
if h_type is None:
raise ValueError("Hash not supported")
base_query = FileWeb.query_find_by_hash(h_type, h_value, None, db)
else:
# FIXME this is just a temporary way to output
# all files, need a dedicated
# file route and controller
base_query = FileWeb.query_find_by_name("", None, db)
# TODO: Find a way to move pagination as a BaseQuery like in
# flask_sqlalchemy.
# https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422
items = base_query.limit(limit).offset(offset).all()
if offset == 0 and len(items) < limit:
total = len(items)
else:
total = base_query.count()
log.debug("Found %s results", total)
response.content_type = "application/json; charset=UTF-8"
return {
'total': total,
'offset': offset,
'limit': limit,
'items': file_web_schema.dump(items, many=True).data,
}
except Exception as e:
log.exception(e)
process_error(e)
def get_stats(db):
""" Search a file using query filters (tags + hash or name). Support
pagination.
:param all params are sent using query method
:rtype: dict of 'total': int, 'page': int, 'per_page': int,
'items': list of file(s) found
:return:
on success 'items' contains a list of files found
on error 'msg' gives reason message
"""
try:
name = None
if 'name' in request.query:
name = decode_utf8(request.query['name'])
h_value = request.query.hash or None
search_tags = request.query.tags or None
if search_tags is not None:
search_tags = search_tags.split(',')
log.debug("name %s h_value %s search_tags %s",
name, h_value, search_tags)
if name is not None and h_value is not None:
raise ValueError("Can't find using both name and hash")
# Options query
offset = int(request.query.offset) if request.query.offset else 0
limit = int(request.query.limit) if request.query.limit else 0
if name is not None:
base_query = FileWeb.query_find_by_name(name, search_tags, db)
elif h_value is not None:
h_type = guess_hash_type(h_value)
if h_type is None:
raise ValueError("Hash not supported")
base_query = FileWeb.query_find_by_hash(
h_type, h_value, search_tags, db)
else:
# FIXME this is just a temporary way to output
# all files, need a dedicated
# file route and controller
base_query = FileWeb.query_find_by_name("", search_tags, db)
# TODO: Find a way to move pagination as a BaseQuery like in
# flask_sqlalchemy.
# https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422
#items = base_query.limit(limit).offset(offset).all()
items = base_query.all()
files_infos = file_web_schema.dump(items, many=True).data
# Process items found statistiques.
#log.debug("Debug ::::::::::::: items : %s", type(items))
#log.debug("Debug ::::::::::::: files_infos: %s", type(files_infos))
stats = []
stats_fake = [
{'name': 'armaditoAV', 'version':3.14, 'nbsamples':2, 'malware': 1, 'clean':1, 'errors':0},
{'name': 'clamav', 'version':3.14, 'nbsamples':2, 'malware': 1, 'clean':1, 'errors':0},
]
for i, val in enumerate(files_infos):
#dsds
log.debug("Debug :::::::::::::: results : %s :: %s", type(val['probe_results']), val['probe_results'])
#log.debug("Debug :::::::::::::: results : %s", type(val.probe_results))
probe_results = val['probe_results']
for j, res in enumerate(probe_results):
#log.debug("Debug :::::::::::::: probe_result : %s", type(res))
# Get av name
#log.debug("Debug :::::::::::::: av_name : %s", res.name)
#log.debug("Debug :::::::::::::: av_type : %s", res.type)
#log.debug("Debug :::::::::::::: av_version : %s", res.version)
if res.type == "antivirus":
add_stats(stats,res)
if offset == 0 and len(items) < limit:
total = len(items)
else:
total = base_query.count()
log.debug("Found %s results", total)
response.content_type = "application/json; charset=UTF-8"
return {
'total': total,
'offset': offset,
'limit': limit,
'items': stats,
}
except Exception as e:
log.exception(e)
process_error(e)
def get_archive(db):
""" Search a file using query filters (tags + hash or name). Support
pagination.
:param all params are sent using query method
:rtype: dict of 'total': int, 'page': int, 'per_page': int,
'items': list of file(s) found
:return:
on success 'items' contains a list of files found
on error 'msg' gives reason message
"""
try:
name = None
if 'name' in request.query:
name = decode_utf8(request.query['name'])
h_value = request.query.hash or None
search_tags = request.query.tags or None
if search_tags is not None:
search_tags = search_tags.split(',')
log.debug("name %s h_value %s search_tags %s",
name, h_value, search_tags)
if name is not None and h_value is not None:
raise ValueError("Can't find using both name and hash")
# Options query
offset = int(request.query.offset) if request.query.offset else 0
limit = int(request.query.limit) if request.query.limit else 25
if name is not None:
base_query = FileWeb.query_find_by_name(name, search_tags, db)
elif h_value is not None:
h_type = guess_hash_type(h_value)
if h_type is None:
raise ValueError("Hash not supported")
base_query = FileWeb.query_find_by_hash(
h_type, h_value, search_tags, db)
else:
# FIXME this is just a temporary way to output
# all files, need a dedicated
# file route and controller
base_query = FileWeb.query_find_by_name("", search_tags, db)
# TODO: Find a way to move pagination as a BaseQuery like in
# flask_sqlalchemy.
# https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422
items = base_query.limit(limit).offset(offset).all()
# get file basic information
file_web_schema = FileWebSchema_v1_1(exclude=('probe_results','file_infos'))
infos = file_web_schema.dump(items, many=True).data
sha256_list = []
for i, val in enumerate(items):
#log.debug("Debug :: items[%s] = %s ::",i, val.file.sha256)
fhash = val.file.sha256
sha256_list.append(fhash)
#file_web.file.sha256
if sha256_list is not None:
return _download_zip(sha256_list,db,infos)
except Exception as e:
log.exception(e)
process_error(e)
import sys
from frontend.models.sqlobjects import Tag
from frontend.helpers.sessions import session_transaction
from lib.common.utils import decode_utf8
if len(sys.argv) != 2:
print("usage: {0} <tag_list> (comma separated)".format(sys.argv[0]))
sys.exit(1)
# get tag list as argument
tag_list = sys.argv[1]
# split comma separated list
tags = tag_list.split(",")
# force all tags to lowercase
tags = map(lambda x: decode_utf8(x.lower()), tags)
with session_transaction() as session:
# get all existing tags
existing_tags = Tag.query_find_all(session)
existing_text = [t.text for t in existing_tags]
# filter only the one needed to be created
to_create_tags = filter(lambda x: x not in existing_text, tags)
print u"[+] Tags already existing: {0}".format(",".join(existing_text))
for tag in to_create_tags:
t = Tag(tag)
print u"[+] creating Tag: {0}".format(tag)
session.add(t)
def files(db):
""" Search a file using query filters (hash or name). Support
pagination.
:param all params are send using query method
:rtype: dict of 'total': int, 'page': int, 'per_page': int,
'items': list of file(s) found
:return:
on success 'items' contains a list of files found
on error 'msg' gives reason message
"""
try:
name = None
if 'name' in request.query:
name = decode_utf8(request.query['name'])
h_value = request.query.hash or None
log.debug("name %s h_value %s", name, h_value)
if name is not None and h_value is not None:
raise ValueError("Can't find using both name and hash")
# Options query
offset = int(request.query.offset) if request.query.offset else 0
limit = int(request.query.limit) if request.query.limit else 25
if name is not None:
base_query = FileWeb.query_find_by_name(name, None, db)
elif h_value is not None:
h_type = guess_hash_type(h_value)
if h_type is None:
raise ValueError("Hash not supported")
base_query = FileWeb.query_find_by_hash(h_type, h_value, None, db)
else:
# FIXME this is just a temporary way to output
# all files, need a dedicated
# file route and controller
base_query = FileWeb.query_find_by_name("", None, db)
# TODO: Find a way to move pagination as a BaseQuery like in
# flask_sqlalchemy.
# https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422
items = base_query.limit(limit).offset(offset).all()
if offset == 0 and len(items) < limit:
total = len(items)
else:
total = base_query.count()
log.debug("Found %s results", total)
response.content_type = "application/json; charset=UTF-8"
return {
'total': total,
'offset': offset,
'limit': limit,
'items': file_web_schema.dump(items, many=True).data,
}
except Exception as e:
log.exception(e)
process_error(e)
