def new(db): """ Add a new tag, text is passed as query parameter. :return: on success new tag object is returned on error 'msg' gives reason message """ try: # Check if parameter is set if 'text' not in request.json: raise ValueError("Missing text parameter") text = request.json['text'] # Check if a tag with same tag already exists available_tags = Tag.query_find_all(db) if text in [t.text for t in available_tags]: raise ValueError("Tag already exists") # create tag if not already present tag = Tag(text) db.add(tag) db.commit() # returns new tag response.content_type = "application/json; charset=UTF-8" return tag_schema.dumps(tag).data except Exception as e: process_error(e)
def list(db): """ Get a list of all scans which have been launched. """ try: # Get values from query or set to default offset = request.query.get("offset", default=0) offset = int(offset) limit = request.query.get("limit", default=5) limit = int(limit) log.debug("offset %s limit %s", offset, limit) base_query = Scan.query_joined(db) items = base_query.limit(limit).offset(offset).all() if offset == 0 and len(items) < limit: total = len(items) else: total = base_query.count() log.debug("found %s scans", total) response.content_type = "application/json; charset=UTF-8" return { "total": total, "offset": offset, "limit": limit, "data": scan_schema.dump(items, many=True).data, } except Exception as e: log.exception(e) process_error(e)
def add_files(scanid, db): """ Attach a file to a scan. The request should be performed using a POST request method. """ try: log.debug("scanid %s", scanid) validate_id(scanid) scan = Scan.load_from_ext_id(scanid, db) if len(request.files) == 0: raise ValueError("No files uploaded") files = {} for f in request.files: upfile = request.files.get(f) filename = decode_utf8(upfile.raw_filename) data = upfile.file files[filename] = data scan_ctrl.add_files(scan, files, db) response.content_type = "application/json; charset=UTF-8" return scan_schema.dumps(scan).data except Exception as e: log.exception(e) process_error(e)
def get(sha256, db): """ Detail about one file and all known scans summary where file was present (identified by sha256). Support pagination. :param all params are sent using query method :param if alt parameter is "media", response will contains the binary data :rtype: dict of 'total': int, 'page': int, 'per_page': int, :return: on success fileinfo contains file information on success 'items' contains a list of files found on error 'msg' gives reason message """ try: log.debug("h_value %s", sha256) # Check wether its a download attempt or not if request.query.alt == "media": return _download(sha256, db) # Get values from query or default offset = request.query.get("offset", default=0) offset = int(offset) limit = request.query.get("limit", default=25) limit = int(limit) file = File.load_from_sha256(sha256, db) # query all known results not only those with different names base_query = FileWeb.query_find_by_hash("sha256", sha256, None, db, distinct_name=False) # TODO: Find a way to move pagination as a BaseQuery like in # flask_sqlalchemy. # https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422 items = base_query.limit(limit).offset(offset).all() if offset == 0 and len(items) < limit: total = len(items) else: total = base_query.count() log.debug("offset %d limit %d total %d", offset, limit, total) file_web_schema = FileWebSchema_v1_1(exclude=('probe_results', 'file_infos')) fileinfo_schema = FileSchema_v1_1() # TODO: allow formatted to be a parameter formatted = True fileinfo_schema.context = {'formatted': formatted} response.content_type = "application/json; charset=UTF-8" return { 'file_infos': fileinfo_schema.dump(file).data, 'total': total, 'offset': offset, 'limit': limit, 'items': file_web_schema.dump(items, many=True).data, } except Exception as e: log.exception(e) process_error(e)
def test004_process_database_error(self, m_sys, m_abort): m_sys.exc_info.return_value = (MagicMock(), None, MagicMock()) error = IrmaDatabaseError("message_error") exp_code = 402 exp_error = api_errors.ApiError("database_error") api_errors.process_error(error) m_abort.assert_called_once() call_args = m_abort.call_args[0] self.assertEqual(call_args[0], exp_code) self.assertEqual(str(call_args[1]), str(exp_error))
def test003_process_notfound_error(self, m_sys, m_abort): m_sys.exc_info.return_value = (MagicMock(), None, MagicMock()) error = IrmaDatabaseResultNotFound("message_error") exp_code = 404 exp_error = api_errors.ApiError("request_error", "Object not Found") api_errors.process_error(error) m_abort.assert_called_once() call_args = m_abort.call_args[0] self.assertEqual(call_args[0], exp_code) self.assertEqual(str(call_args[1]), str(exp_error))
def remove_tag(sha256, tagid, db): """ Remove a tag attached to a file. """ try: log.debug("h_value %s tagid %s", sha256, tagid) fobj = File.load_from_sha256(sha256, db) fobj.remove_tag(tagid, db) db.commit() except Exception as e: log.exception(e) process_error(e)
def add_tag(sha256, tagid, db): """ Attach a tag to a file. """ try: log.debug("h_value %s tagid %s", sha256, tagid) fobj = File.load_from_sha256(sha256, db) fobj.add_tag(tagid, db) db.commit() except Exception as e: log.exception(e) process_error(e)
def list(): """ get active probe list. This list is used to launch a scan. """ try: probelist = celery_brain.probe_list() log.debug("probe list: %s", "-".join(probelist)) response.content_type = "application/json; charset=UTF-8" return {"total": len(probelist), "data": probelist} except Exception as e: log.exception(e) process_error(e)
def get(scanid, db): """ Retrieve information for a specific scan """ try: log.debug("scanid %s", scanid) validate_id(scanid) scan = Scan.load_from_ext_id(scanid, db) response.content_type = "application/json; charset=UTF-8" return scan_schema.dumps(scan).data except Exception as e: log.exception(e) process_error(e)
def list(): """ get active probe list. This list is used to launch a scan. """ try: probelist = celery_brain.probe_list() log.debug("probe list: %s", "-".join(probelist)) response.content_type = "application/json; charset=UTF-8" return { "total": len(probelist), "data": probelist } except Exception as e: log.exception(e) process_error(e)
def list(db): """ Search for all tags in TAG table using query. :return: on success 'items' contains a list of all tags on error 'msg' gives reason message """ try: available_tags = Tag.query_find_all(db) response.content_type = "application/json; charset=UTF-8" return { 'items': tag_schema.dump(available_tags, many=True).data, } except Exception as e: process_error(e)
def cancel(scanid, db): """ Cancel a scan. The request should be performed using a POST request method. """ try: log.debug("scanid %s", scanid) validate_id(scanid) scan = Scan.load_from_ext_id(scanid, db) scan_ctrl.cancel(scan, db) response.content_type = "application/json; charset=UTF-8" return scan_schema.dumps(scan).data except Exception as e: log.exception(e) process_error(e)
def get_results(scanid, db): """ Retrieve results for a scan. Results are the same as in the get() method, i.e. a summary for each scanned files. The request should be performed using a GET request method. """ try: log.debug("scanid %s", scanid) validate_id(scanid) scan = Scan.load_from_ext_id(scanid, db) file_web_schema = FileWebSchema_v1_1(exclude=('probe_results', 'file_infos')) response.content_type = "application/json; charset=UTF-8" return file_web_schema.dumps(scan.files_web, many=True).data except Exception as e: log.exception(e) process_error(e)
def new(db): """ Create a new scan. The request should be performed using a POST request method. """ try: ip = request.remote_addr scan = Scan(compat.timestamp(), ip) db.add(scan) scan.set_status(IrmaScanStatus.empty) db.commit() log.debug("scanid %s", scan.external_id) response.content_type = "application/json; charset=UTF-8" return scan_schema.dumps(scan).data except Exception as e: log.exception(e) process_error(e)
def launch(scanid, db): """ Launch a scan. The request should be performed using a POST request method. """ try: validate_id(scanid) scan = Scan.load_from_ext_id(scanid, db) probes = None # handle scan parameter # cached results: "force" (default: True) scan.force = False if 'force' in request.json and request.json.get('force') is True: scan.force = True # use mimetype for probelist: "mimetype_filtering" (default: True) scan.mimetype_filtering = True if 'mimetype_filtering' in request.json and \ request.json.get('mimetype_filtering') is False: scan.mimetype_filtering = False # rescan file outputted from probes "resubmit_files" (default: True) scan.resubmit_files = True if 'resubmit_files' in request.json and \ request.json.get('resubmit_files') is False: scan.resubmit_files = False db.commit() # handle scan parameter / probelist: "probes" if 'probes' in request.json: probes = request.json.get('probes').split(',') msg = "scanid %s Force %s MimeF %s" msg += "Resub %s Probes %s" log.debug(msg, scanid, scan.force, scan.mimetype_filtering, scan.resubmit_files, probes) scan_ctrl.check_probe(scan, probes, db) # launch_asynchronous scan via frontend task celery_frontend.scan_launch(scanid) response.content_type = "application/json; charset=UTF-8" return scan_schema.dumps(scan).data except Exception as e: log.exception(e) process_error(e)
def get(resultid, db): """ Retrieve a single fileweb result, with details. The request should be performed using a GET request method. """ try: formatted = False if request.query.formatted == 'no' else True log.debug("resultid %s formatted %s", resultid, formatted) validate_id(resultid) fw = FileWeb.load_from_ext_id(resultid, db) file_web_schema = FileWebSchema_v1_1() file_web_schema.context = {'formatted': formatted} response.content_type = "application/json; charset=UTF-8" return file_web_schema.dumps(fw).data except Exception as e: log.exception(e) process_error(e)
def get_stats(db): """ Search a file using query filters (tags + hash or name). Support pagination. :param all params are sent using query method :rtype: dict of 'total': int, 'page': int, 'per_page': int, 'items': list of file(s) found :return: on success 'items' contains a list of files found on error 'msg' gives reason message """ try: name = None if 'name' in request.query: name = decode_utf8(request.query['name']) h_value = request.query.hash or None search_tags = request.query.tags or None if search_tags is not None: search_tags = search_tags.split(',') log.debug("name %s h_value %s search_tags %s", name, h_value, search_tags) if name is not None and h_value is not None: raise ValueError("Can't find using both name and hash") # Options query offset = int(request.query.offset) if request.query.offset else 0 limit = int(request.query.limit) if request.query.limit else 0 if name is not None: base_query = FileWeb.query_find_by_name(name, search_tags, db) elif h_value is not None: h_type = guess_hash_type(h_value) if h_type is None: raise ValueError("Hash not supported") base_query = FileWeb.query_find_by_hash( h_type, h_value, search_tags, db) else: # FIXME this is just a temporary way to output # all files, need a dedicated # file route and controller base_query = FileWeb.query_find_by_name("", search_tags, db) # TODO: Find a way to move pagination as a BaseQuery like in # flask_sqlalchemy. # https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422 #items = base_query.limit(limit).offset(offset).all() items = base_query.all() files_infos = file_web_schema.dump(items, many=True).data # Process items found statistiques. #log.debug("Debug ::::::::::::: items : %s", type(items)) #log.debug("Debug ::::::::::::: files_infos: %s", type(files_infos)) stats = [] stats_fake = [ {'name': 'armaditoAV', 'version':3.14, 'nbsamples':2, 'malware': 1, 'clean':1, 'errors':0}, {'name': 'clamav', 'version':3.14, 'nbsamples':2, 'malware': 1, 'clean':1, 'errors':0}, ] for i, val in enumerate(files_infos): #dsds log.debug("Debug :::::::::::::: results : %s :: %s", type(val['probe_results']), val['probe_results']) #log.debug("Debug :::::::::::::: results : %s", type(val.probe_results)) probe_results = val['probe_results'] for j, res in enumerate(probe_results): #log.debug("Debug :::::::::::::: probe_result : %s", type(res)) # Get av name #log.debug("Debug :::::::::::::: av_name : %s", res.name) #log.debug("Debug :::::::::::::: av_type : %s", res.type) #log.debug("Debug :::::::::::::: av_version : %s", res.version) if res.type == "antivirus": add_stats(stats,res) if offset == 0 and len(items) < limit: total = len(items) else: total = base_query.count() log.debug("Found %s results", total) response.content_type = "application/json; charset=UTF-8" return { 'total': total, 'offset': offset, 'limit': limit, 'items': stats, } except Exception as e: log.exception(e) process_error(e)
def list(db): """ Search a file using query filters (tags + hash or name). Support pagination. :param all params are sent using query method :rtype: dict of 'total': int, 'page': int, 'per_page': int, 'items': list of file(s) found :return: on success 'items' contains a list of files found on error 'msg' gives reason message """ try: name = None if 'name' in request.query: name = decode_utf8(request.query['name']) h_value = request.query.hash or None search_tags = request.query.tags or None if search_tags is not None: search_tags = search_tags.split(',') log.debug("name %s h_value %s search_tags %s", name, h_value, search_tags) if name is not None and h_value is not None: raise ValueError("Can't find using both name and hash") # Options query offset = int(request.query.offset) if request.query.offset else 0 limit = int(request.query.limit) if request.query.limit else 25 if name is not None: base_query = FileWeb.query_find_by_name(name, search_tags, db) elif h_value is not None: h_type = guess_hash_type(h_value) if h_type is None: raise ValueError("Hash not supported") base_query = FileWeb.query_find_by_hash(h_type, h_value, search_tags, db) else: # FIXME this is just a temporary way to output # all files, need a dedicated # file route and controller base_query = FileWeb.query_find_by_name("", search_tags, db) # TODO: Find a way to move pagination as a BaseQuery like in # flask_sqlalchemy. # https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422 items = base_query.limit(limit).offset(offset).all() if offset == 0 and len(items) < limit: total = len(items) else: total = base_query.count() log.debug("Found %s results", total) response.content_type = "application/json; charset=UTF-8" return { 'total': total, 'offset': offset, 'limit': limit, 'items': file_web_schema.dump(items, many=True).data, } except Exception as e: log.exception(e) process_error(e)
def get_archive(db): """ Search a file using query filters (tags + hash or name). Support pagination. :param all params are sent using query method :rtype: dict of 'total': int, 'page': int, 'per_page': int, 'items': list of file(s) found :return: on success 'items' contains a list of files found on error 'msg' gives reason message """ try: name = None if 'name' in request.query: name = decode_utf8(request.query['name']) h_value = request.query.hash or None search_tags = request.query.tags or None if search_tags is not None: search_tags = search_tags.split(',') log.debug("name %s h_value %s search_tags %s", name, h_value, search_tags) if name is not None and h_value is not None: raise ValueError("Can't find using both name and hash") # Options query offset = int(request.query.offset) if request.query.offset else 0 limit = int(request.query.limit) if request.query.limit else 25 if name is not None: base_query = FileWeb.query_find_by_name(name, search_tags, db) elif h_value is not None: h_type = guess_hash_type(h_value) if h_type is None: raise ValueError("Hash not supported") base_query = FileWeb.query_find_by_hash( h_type, h_value, search_tags, db) else: # FIXME this is just a temporary way to output # all files, need a dedicated # file route and controller base_query = FileWeb.query_find_by_name("", search_tags, db) # TODO: Find a way to move pagination as a BaseQuery like in # flask_sqlalchemy. # https://github.com/mitsuhiko/flask-sqlalchemy/blob/master/flask_sqlalchemy/__init__.py#L422 items = base_query.limit(limit).offset(offset).all() # get file basic information file_web_schema = FileWebSchema_v1_1(exclude=('probe_results','file_infos')) infos = file_web_schema.dump(items, many=True).data sha256_list = [] for i, val in enumerate(items): #log.debug("Debug :: items[%s] = %s ::",i, val.file.sha256) fhash = val.file.sha256 sha256_list.append(fhash) #file_web.file.sha256 if sha256_list is not None: return _download_zip(sha256_list,db,infos) except Exception as e: log.exception(e) process_error(e)