def test_set_user_settings(datastore, login_session): _, session, host = login_session username = random.choice(user_list) uset = load_user_settings({'uname': username}) uset['expand_min_score'] = 111 uset['priority'] = 111 resp = get_api_data(session, f"{host}/api/v4/user/settings/{username}/", method="POST", data=json.dumps(uset)) assert resp['success'] datastore.user_settings.commit() assert uset == load_user_settings({'uname': username})
def get_user_settings(username, **kwargs): """ Load the user's settings. Variables: username => Name of the user you want to get the settings for Arguments: None Data Block: None Result example: { "profile": true, # Should submissions be profiled "classification": "", # Default classification for this user sumbissions "description": "", # Default description for this user's submissions "download_encoding": "blah", # Default encoding for downloaded files "expand_min_score": 100, # Default minimum score to auto-expand sections "priority": 1000, # Default submission priority "service_spec": [], # Default Service specific parameters "ignore_cache": true, # Should file be reprocessed even if there are cached results "groups": [ ... ], # Default groups selection for the user scans "ttl": 30, # Default time to live in days of the users submissions "services": [ ... ], # Default list of selected services "ignore_filtering": false # Should filtering services by ignored? } """ user = kwargs['user'] if username != user['uname']: user = STORAGE.user.get(username, as_obj=False) return make_api_response(load_user_settings(user))
def get_user_submission_params(username, **kwargs): """ Load the user's default submission params that should be passed to the submit API. This is mainly use so you can alter a couple fields and preserve the user default values. Variables: username => Name of the user you want to get the settings for Arguments: None Data Block: None Result example: { "profile": true, # Should submissions be profiled "classification": "", # Default classification for this user sumbissions "description": "", # Default description for this user's submissions "priority": 1000, # Default submission priority "service_spec": [], # Default Service specific parameters "ignore_cache": true, # Should file be reprocessed even if there are cached results "groups": [ ... ], # Default groups selection for the user scans "ttl": 30, # Default time to live in days of the users submissions "services": [ ... ], # Default list of selected services "ignore_filtering": false # Should filtering services by ignored? } """ user = kwargs['user'] if username != "__CURRENT__" and username != user['uname']: user = STORAGE.user.get(username, as_obj=False) params = load_user_settings(user) submission_params = ui_to_submission_params(params) submission_params['submitter'] = username submission_params['groups'] = user['groups'] return make_api_response(submission_params)
def download_file(sha256, **kwargs): """ Download the file using the default encoding method. This api will force the browser in download mode. Variables: sha256 => A resource locator for the file (sha256) Arguments (optional): encoding => Type of encoding use for the resulting file name => Name of the file to download sid => Submission ID where the file is from Data Block: None API call example: /api/v4/file/download/123456...654321/ Result example: <THE FILE BINARY ENCODED IN SPECIFIED FORMAT> """ user = kwargs['user'] file_obj = STORAGE.file.get(sha256, as_obj=False) if not file_obj: return make_api_response({}, "The file was not found in the system.", 404) if user and Classification.is_accessible(user['classification'], file_obj['classification']): params = load_user_settings(user) name = request.args.get('name', sha256) or sha256 name = os.path.basename(name) name = safe_str(name) sid = request.args.get('sid', None) or None submission = {} file_metadata = {} if sid is not None: submission = STORAGE.submission.get(sid, as_obj=False) if submission is None: submission = {} if Classification.is_accessible(user['classification'], submission['classification']): file_metadata.update(unflatten(submission['metadata'])) if Classification.enforce: submission_classification = submission.get( 'classification', file_obj['classification']) file_metadata[ 'classification'] = Classification.max_classification( submission_classification, file_obj['classification']) encoding = request.args.get('encoding', params['download_encoding']) password = request.args.get('password', params['default_zip_password']) if encoding not in FILE_DOWNLOAD_ENCODINGS: return make_api_response( {}, f"{encoding.upper()} is not in the valid encoding types: {FILE_DOWNLOAD_ENCODINGS}", 403) if encoding == "raw" and not ALLOW_RAW_DOWNLOADS: return make_api_response( {}, "RAW file download has been disabled by administrators.", 403) if encoding == "zip": if not ALLOW_ZIP_DOWNLOADS: return make_api_response( {}, "PROTECTED file download has been disabled by administrators.", 403) elif not password: return make_api_response( {}, "No password given or retrieved from user's settings.", 403) download_dir = None target_path = None # Create a temporary download location if encoding == 'zip': download_dir = tempfile.mkdtemp() download_path = os.path.join(download_dir, name) else: _, download_path = tempfile.mkstemp() try: downloaded_from = FILESTORE.download(sha256, download_path) if not downloaded_from: return make_api_response( {}, "The file was not found in the system.", 404) # Encode file if encoding == 'raw': target_path = download_path elif encoding == 'zip': name += '.zip' target_path = os.path.join(download_dir, name) subprocess.run([ 'zip', '-j', '--password', password, target_path, download_path ], capture_output=True) else: target_path, name = encode_file(download_path, name, file_metadata) return stream_file_response(open(target_path, 'rb'), name, os.path.getsize(target_path)) finally: # Cleanup if target_path: if os.path.exists(target_path): os.unlink(target_path) if download_path: if os.path.exists(download_path): os.unlink(download_path) if download_dir: if os.path.exists(download_dir): os.rmdir(download_dir) else: return make_api_response({}, "You are not allowed to download this file.", 403)
def who_am_i(**kwargs): """ Return the currently logged in user as well as the system configuration Variables: None Arguments: None Data Block: None Result example: { "agrees_with_tos": None, # Date the user agreed with TOS "avatar": "data:image/jpg...", # Avatar data block "c12nDef": {}, # Classification definition block "classification": "TLP:W", # Classification of the user "configuration": { # Configuration block "auth": { # Authentication Configuration "allow_2fa": True, # Is 2fa Allowed for the user "allow_apikeys": True, # Are APIKeys allowed for the user "allow_extended_apikeys": True, # Allow user to generate extended access API Keys "allow_security_tokens": True, # Are Security tokens allowed for the user }, "submission": { # Submission Configuration "dtl": 10, # Default number of days submission stay in the system "max_dtl": 30, # Maximum number of days submission stay in the system }, "system": { # System Configuration "organisation": "ACME", # Organisation name "type": "production", # Type of deployment "version": "4.1" # Assemblyline version }, "ui": { # UI Configuration "alerting_meta": { # Alert metadata configuration "important": [], # List of metadata fields that should always be displayed "subject": [], # List of metadata fields where to fetch email subject "url": [] # List of metadata fields where to fetch URLS }, "allow_malicious_hinting": True, # Are users allowed to set the malicious flag before processing "allow_raw_downloads": True, # Are users allowed to download files in their raw format? "allow_zip_downloads": True, # Are users allowed to download files as password-protected ZIPs? "allow_replay": False, # Are users allowed to continue submissions on another server "allow_url_submissions": True, # Are URL submissions allowed "apps": [], # List of apps shown in the apps switcher "banner": None, # Banner displayed on the submit page "banner_level": True, # Banner color (info, success, warning, error) "read_only": False, # Is the interface to be displayed in read-only mode "tos": True, # Are terms of service set in the system "tos_lockout": False, # Will agreeing to TOS lockout the user "tos_lockout_notify": False # Will admin be auto-notified when a user is locked out } }, "email": "*****@*****.**", # Email of the user "groups": ["USERS"], # Groups the user if member of "indexes": {}, # Search indexes definitions "is_active": True, # Is the user active "name": "Basic user", # Name of the user "type": ["user", "admin"], # Roles the user is member of "uname": "sgaron-cyber" # Username of the current user } """ user_data = {k: v for k, v in kwargs['user'].items() if k in ["agrees_with_tos", "classification", "email", "groups", "is_active", "name", "type", "uname"]} user_data['avatar'] = STORAGE.user_avatar.get(kwargs['user']['uname']) user_data['username'] = user_data.pop('uname') user_data['is_admin'] = "admin" in user_data['type'] user_data['roles'] = user_data.pop('type') # System configuration user_data['c12nDef'] = classification_definition user_data['configuration'] = { "auth": { "allow_2fa": config.auth.allow_2fa, "allow_apikeys": config.auth.allow_apikeys, "allow_extended_apikeys": config.auth.allow_extended_apikeys, "allow_security_tokens": config.auth.allow_security_tokens, }, "submission": { "dtl": config.submission.dtl, "max_dtl": config.submission.max_dtl }, "system": { "organisation": config.system.organisation, "type": config.system.type, "version": VERSION }, "ui": { "alerting_meta": { "important": config.ui.alerting_meta.important, "subject": config.ui.alerting_meta.subject, "url": config.ui.alerting_meta.url }, "allow_malicious_hinting": config.ui.allow_malicious_hinting, "allow_raw_downloads": config.ui.allow_raw_downloads, "allow_zip_downloads": config.ui.allow_zip_downloads, "allow_replay": config.ui.allow_replay, "allow_url_submissions": config.ui.allow_url_submissions, "apps": [x for x in APPS_LIST['apps'] if CLASSIFICATION.is_accessible(kwargs['user']['classification'], x['classification'] or CLASSIFICATION.UNRESTRICTED, ignore_invalid=True)], "banner": config.ui.banner, "banner_level": config.ui.banner_level, "read_only": config.ui.read_only, "tos": config.ui.tos not in [None, ""], "tos_lockout": config.ui.tos_lockout, "tos_lockout_notify": config.ui.tos_lockout_notify not in [None, []] }, } user_data['indexes'] = list_all_fields(user_data) user_data['settings'] = load_user_settings(kwargs['user']) msg = UI_MESSAGING.get('system_message') if msg: user_data['system_message'] = msg return make_api_response(user_data)
def ingest_single_file(**kwargs): """ Ingest a single file, sha256 or URL in the system Note 1: If you are submitting a sha256 or a URL, you must use the application/json encoding and one of sha256 or url parameters must be included in the data block. Note 2: If you are submitting a file directly, you have to use multipart/form-data encoding this was done to reduce the memory footprint and speedup file transfers ** Read documentation of mime multipart standard if your library does not support it** The multipart/form-data for sending binary has two parts: - The first part contains a JSON dump of the optional params and uses the name 'json' - The last part conatins the file binary, uses the name 'bin' and includes a filename Note 3: The ingest API uses the user's default settings to submit files to the system unless these settings are overridden in the 'params' field. Although, there are exceptions to that rule. Fields deep_scan, ignore_filtering, ignore_cache are resetted to False because the lead to dangerous behavior in the system. Variables: None Arguments: None Data Block (SHA256 or URL): { //REQUIRED VALUES: One of the following "sha256": "1234...CDEF" # SHA256 hash of the file "url": "http://...", # Url to fetch the file from //OPTIONAL VALUES "name": "file.exe", # Name of the file "metadata": { # Submission Metadata "key": val, # Key/Value pair for metadata parameters }, "params": { # Submission parameters "key": val, # Key/Value pair for params that differ from the user's defaults }, # DEFAULT: /api/v3/user/submission_params/<user>/ "generate_alert": False, # Generate an alert in our alerting system or not "notification_queue": None, # Name of the notification queue "notification_threshold": None, # Threshold for notification } Data Block (Binary): --0b34a3c50d3c02dd804a172329a0b2aa <-- Randomly generated boundary for this http request Content-Disposition: form-data; name="json" <-- JSON data blob part (only previous optional values valid) {"params": {"ignore_cache": true}, "generate_alert": true} --0b34a3c50d3c02dd804a172329a0b2aa <-- Switch to next part, file part Content-Disposition: form-data; name="bin"; filename="name_of_the_file_to_scan.bin" <BINARY DATA OF THE FILE TO SCAN... DOES NOT NEED TO BE ENCODDED> --0b34a3c50d3c02dd804a172329a0b2aa-- <-- End of HTTP transmission Result example: { "ingest_id": <ID OF THE INGESTED FILE> } """ user = kwargs['user'] out_dir = os.path.join(TEMP_SUBMIT_DIR, get_random_id()) extracted_path = original_file = None try: # Get data block and binary blob if 'multipart/form-data' in request.content_type: if 'json' in request.values: data = json.loads(request.values['json']) else: data = {} binary = request.files['bin'] name = data.get("name", binary.filename) sha256 = None url = None elif 'application/json' in request.content_type: data = request.json binary = None sha256 = data.get('sha256', None) url = data.get('url', None) name = data.get("name", None) or sha256 or os.path.basename(url) or None else: return make_api_response({}, "Invalid content type", 400) if not data: return make_api_response({}, "Missing data block", 400) # Get notification queue parameters notification_queue = data.get('notification_queue', None) notification_threshold = data.get('notification_threshold', None) if not isinstance(notification_threshold, int) and notification_threshold: return make_api_response( {}, "notification_threshold should be and int", 400) # Get file name if not name: return make_api_response({}, "Filename missing", 400) name = safe_str(os.path.basename(name)) if not name: return make_api_response({}, "Invalid filename", 400) try: os.makedirs(out_dir) except Exception: pass original_file = out_file = os.path.join(out_dir, name) # Prepare variables extra_meta = {} fileinfo = None do_upload = True al_meta = {} # Load default user params s_params = ui_to_submission_params(load_user_settings(user)) # Reset dangerous user settings to safe values s_params.update({ 'deep_scan': False, "priority": 150, "ignore_cache": False, "ignore_dynamic_recursion_prevention": False, "ignore_filtering": False, "type": "INGEST" }) # Apply provided params s_params.update(data.get("params", {})) # Load file if not binary: if sha256: fileinfo = STORAGE.file.get_if_exists( sha256, as_obj=False, archive_access=config.datastore.ilm.update_archive) if FILESTORE.exists(sha256): if fileinfo: if not Classification.is_accessible( user['classification'], fileinfo['classification']): return make_api_response( {}, "SHA256 does not exist in our datastore", 404) else: # File's classification must be applied at a minimum s_params[ 'classification'] = Classification.max_classification( s_params['classification'], fileinfo['classification']) else: # File is in storage and the DB no need to upload anymore do_upload = False # File exists in the filestore and the user has appropriate file access FILESTORE.download(sha256, out_file) else: return make_api_response( {}, "SHA256 does not exist in our datastore", 404) else: if url: if not config.ui.allow_url_submissions: return make_api_response( {}, "URL submissions are disabled in this system", 400) try: safe_download(url, out_file) extra_meta['submitted_url'] = url except FileTooBigException: return make_api_response({}, "File too big to be scanned.", 400) except InvalidUrlException: return make_api_response({}, "Url provided is invalid.", 400) except ForbiddenLocation: return make_api_response( {}, "Hostname in this URL cannot be resolved.", 400) else: return make_api_response( {}, "Missing file to scan. No binary, sha256 or url provided.", 400) else: binary.save(out_file) if do_upload and os.path.getsize(out_file) == 0: return make_api_response({}, err="File empty. Ingestion failed", status_code=400) # Apply group params if not specified if 'groups' not in s_params: s_params['groups'] = user['groups'] # Get generate alert parameter generate_alert = data.get('generate_alert', s_params.get('generate_alert', False)) if not isinstance(generate_alert, bool): return make_api_response({}, "generate_alert should be a boolean", 400) # Override final parameters s_params.update({ 'generate_alert': generate_alert, 'max_extracted': config.core.ingester.default_max_extracted, 'max_supplementary': config.core.ingester.default_max_supplementary, 'priority': min(s_params.get("priority", 150), config.ui.ingest_max_priority), 'submitter': user['uname'] }) # Enforce maximum DTL if config.submission.max_dtl > 0: s_params['ttl'] = min(int( s_params['ttl']), config.submission.max_dtl) if int( s_params['ttl']) else config.submission.max_dtl # No need to re-calculate fileinfo if we have it already if not fileinfo: # Calculate file digest fileinfo = IDENTIFY.fileinfo(out_file) # Validate file size if fileinfo['size'] > MAX_SIZE and not s_params.get( 'ignore_size', False): msg = f"File too large ({fileinfo['size']} > {MAX_SIZE}). Ingestion failed" return make_api_response({}, err=msg, status_code=413) elif fileinfo['size'] == 0: return make_api_response({}, err="File empty. Ingestion failed", status_code=400) # Decode cart if needed extracted_path, fileinfo, al_meta = decode_file( out_file, fileinfo, IDENTIFY) if extracted_path: out_file = extracted_path # Alter filename and classification based on CaRT output meta_classification = al_meta.pop('classification', s_params['classification']) if meta_classification != s_params['classification']: try: s_params['classification'] = Classification.max_classification( meta_classification, s_params['classification']) except InvalidClassification as ic: return make_api_response( {}, "The classification found inside the cart file cannot be merged with " f"the classification the file was submitted as: {str(ic)}", 400) name = al_meta.pop('name', name) # Validate ingest classification if not Classification.is_accessible(user['classification'], s_params['classification']): return make_api_response( {}, "You cannot start a submission with higher " "classification then you're allowed to see", 400) # Freshen file object expiry = now_as_iso(s_params['ttl'] * 24 * 60 * 60) if s_params.get('ttl', None) else None STORAGE.save_or_freshen_file(fileinfo['sha256'], fileinfo, expiry, s_params['classification']) # Save the file to the filestore if needs be # also no need to test if exist before upload because it already does that if do_upload: FILESTORE.upload(out_file, fileinfo['sha256'], location='far') # Setup notification queue if needed if notification_queue: notification_params = { "queue": notification_queue, "threshold": notification_threshold } else: notification_params = {} # Load metadata, setup some default values if they are missing and append the cart metadata ingest_id = get_random_id() metadata = flatten(data.get("metadata", {})) metadata['ingest_id'] = ingest_id metadata['type'] = s_params['type'] metadata.update(al_meta) if 'ts' not in metadata: metadata['ts'] = now_as_iso() metadata.update(extra_meta) # Set description if it does not exists s_params['description'] = s_params[ 'description'] or f"[{s_params['type']}] Inspection of file: {name}" # Create submission object try: submission_obj = Submission({ "sid": ingest_id, "files": [{ 'name': name, 'sha256': fileinfo['sha256'], 'size': fileinfo['size'] }], "notification": notification_params, "metadata": metadata, "params": s_params }) except (ValueError, KeyError) as e: return make_api_response({}, err=str(e), status_code=400) # Send submission object for processing ingest.push(submission_obj.as_primitives()) submission_received(submission_obj) return make_api_response({"ingest_id": ingest_id}) finally: # Cleanup files on disk try: if original_file and os.path.exists(original_file): os.unlink(original_file) except Exception: pass try: if extracted_path and os.path.exists(extracted_path): os.unlink(extracted_path) except Exception: pass try: if os.path.exists(out_dir): shutil.rmtree(out_dir, ignore_errors=True) except Exception: pass
def resubmit_for_dynamic(sha256, *args, **kwargs): """ Resubmit a file for dynamic analysis Variables: sha256 => Resource locator (SHA256) Arguments (Optional): copy_sid => Mimic the attributes of this SID. name => Name of the file for the submission Data Block: None Result example: # Submission message object as a json dictionary """ user = kwargs['user'] quota_error = check_submission_quota(user) if quota_error: return make_api_response("", quota_error, 503) submit_result = None try: copy_sid = request.args.get('copy_sid', None) name = safe_str(request.args.get('name', sha256)) if copy_sid: submission = STORAGE.submission.get(copy_sid, as_obj=False) else: submission = None if submission: if not Classification.is_accessible(user['classification'], submission['classification']): return make_api_response( "", "You are not allowed to re-submit a submission that you don't have access to", 403) submission_params = submission['params'] submission_params['classification'] = submission['classification'] else: submission_params = ui_to_submission_params( load_user_settings(user)) with forge.get_filestore() as f_transport: if not f_transport.exists(sha256): return make_api_response( {}, "File %s cannot be found on the server therefore it cannot be resubmitted." % sha256, status_code=404) files = [{'name': name, 'sha256': sha256}] submission_params['submitter'] = user['uname'] submission_params['quota_item'] = True if 'priority' not in submission_params: submission_params['priority'] = 500 submission_params[ 'description'] = "Resubmit %s for Dynamic Analysis" % name if "Dynamic Analysis" not in submission_params['services'][ 'selected']: submission_params['services']['selected'].append( "Dynamic Analysis") try: submission_obj = Submission({ "files": files, "params": submission_params }) except (ValueError, KeyError) as e: return make_api_response("", err=str(e), status_code=400) submit_result = SubmissionClient( datastore=STORAGE, filestore=f_transport, config=config).submit(submission_obj) submission_received(submission_obj) return make_api_response(submit_result.as_primitives()) except SubmissionException as e: return make_api_response("", err=str(e), status_code=400) finally: if submit_result is None: decrement_submission_quota(user)
def submit(**kwargs): """ Submit a single file, sha256 or url for analysis Note 1: If you are submitting a sh256 or a URL, you must use the application/json encoding and one of sha256 or url parameters must be included in the data block. Note 2: If you are submitting a file directly, you have to use multipart/form-data encoding this was done to reduce the memory footprint and speedup file transfers ** Read documentation of mime multipart standard if your library does not support it** The multipart/form-data for sending binary has two parts: - The first part contains a JSON dump of the optional params and uses the name 'json' - The last part conatins the file binary, uses the name 'bin' and includes a filename Variables: None Arguments: None Data Block (SHA256 or URL): { // REQUIRED: One of the two following "sha256": "123...DEF", # SHA256 hash of the file already in the datastore "url": "http://...", # Url to fetch the file from // OPTIONAL VALUES "name": "file.exe", # Name of the file to scan otherwise the sha256 or base file of the url "metadata": { # Submission metadata "key": val, # Key/Value pair metadata values }, "params": { # Submission parameters "key": val, # Key/Value pair for params that different then defaults }, # Default params can be fetch at /api/v3/user/submission_params/<user>/ } Data Block (Binary): --0b34a3c50d3c02dd804a172329a0b2aa <-- Randomly generated boundary for this http request Content-Disposition: form-data; name="json" <-- JSON data blob part (only previous optional values valid) {"metadata": {"hello": "world"}} --0b34a3c50d3c02dd804a172329a0b2aa <-- Switch to next part, file part Content-Disposition: form-data; name="bin"; filename="name_of_the_file_to_scan.bin" <BINARY DATA OF THE FILE TO SCAN... DOES NOT NEED TO BE ENCODDED> --0b34a3c50d3c02dd804a172329a0b2aa-- <-- End of HTTP transmission Result example: <Submission message object as a json dictionary> """ user = kwargs['user'] out_dir = os.path.join(TEMP_SUBMIT_DIR, get_random_id()) with forge.get_filestore() as f_transport: quota_error = check_submission_quota(user) if quota_error: return make_api_response("", quota_error, 503) submit_result = None try: # Get data block and binary blob if 'multipart/form-data' in request.content_type: if 'json' in request.values: data = json.loads(request.values['json']) else: data = {} binary = request.files['bin'] name = data.get("name", binary.filename) sha256 = None url = None elif 'application/json' in request.content_type: data = request.json binary = None sha256 = data.get('sha256', None) url = data.get('url', None) name = data.get( "name", None) or sha256 or os.path.basename(url) or None else: return make_api_response({}, "Invalid content type", 400) if data is None: return make_api_response({}, "Missing data block", 400) if not name: return make_api_response({}, "Filename missing", 400) name = safe_str(os.path.basename(name)) if not name: return make_api_response({}, "Invalid filename", 400) # Create task object if "ui_params" in data: s_params = ui_to_submission_params(data['ui_params']) else: s_params = ui_to_submission_params(load_user_settings(user)) s_params.update(data.get("params", {})) if 'groups' not in s_params: s_params['groups'] = user['groups'] s_params['quota_item'] = True s_params['submitter'] = user['uname'] if not s_params['description']: s_params['description'] = "Inspection of file: %s" % name if not Classification.is_accessible(user['classification'], s_params['classification']): return make_api_response( {}, "You cannot start a scan with higher " "classification then you're allowed to see", 400) # Prepare the output directory try: os.makedirs(out_dir) except Exception: pass out_file = os.path.join(out_dir, name) # Get the output file extra_meta = {} if not binary: if sha256: if f_transport.exists(sha256): f_transport.download(sha256, out_file) else: return make_api_response( {}, "SHA256 does not exist in our datastore", 404) else: if url: if not config.ui.allow_url_submissions: return make_api_response( {}, "URL submissions are disabled in this system", 400) try: safe_download(url, out_file) extra_meta['submitted_url'] = url except FileTooBigException: return make_api_response( {}, "File too big to be scanned.", 400) except InvalidUrlException: return make_api_response( {}, "Url provided is invalid.", 400) except ForbiddenLocation: return make_api_response( {}, "Hostname in this URL cannot be resolved.", 400) else: return make_api_response( {}, "Missing file to scan. No binary, sha256 or url provided.", 400) else: with open(out_file, "wb") as my_file: my_file.write(binary.read()) try: metadata = flatten(data.get('metadata', {})) metadata.update(extra_meta) submission_obj = Submission({ "files": [], "metadata": metadata, "params": s_params }) except (ValueError, KeyError) as e: return make_api_response("", err=str(e), status_code=400) # Submit the task to the system try: submit_result = SubmissionClient(datastore=STORAGE, filestore=f_transport, config=config)\ .submit(submission_obj, local_files=[out_file], cleanup=False) submission_received(submission_obj) except SubmissionException as e: return make_api_response("", err=str(e), status_code=400) return make_api_response(submit_result.as_primitives()) finally: if submit_result is None: decrement_submission_quota(user) try: # noinspection PyUnboundLocalVariable os.unlink(out_file) except Exception: pass try: shutil.rmtree(out_dir, ignore_errors=True) except Exception: pass
def download_file(sha256, **kwargs): """ Download the file using the default encoding method. This api will force the browser in download mode. Variables: sha256 => A resource locator for the file (sha256) Arguments (optional): encoding => Type of encoding use for the resulting file name => Name of the file to download sid => Submission ID where the file is from Data Block: None API call example: /api/v4/file/download/123456...654321/ Result example: <THE FILE BINARY ENCODED IN SPECIFIED FORMAT> """ user = kwargs['user'] file_obj = STORAGE.file.get(sha256, as_obj=False) if not file_obj: return make_api_response({}, "The file was not found in the system.", 404) if user and Classification.is_accessible(user['classification'], file_obj['classification']): params = load_user_settings(user) name = request.args.get('name', sha256) or sha256 name = os.path.basename(name) name = safe_str(name) sid = request.args.get('sid', None) or None submission = {} submission_meta = {} if sid is not None: submission = STORAGE.submission.get(sid, as_obj=False) if submission is None: submission = {} hash_list = [submission.get('files', [])[0].get('sha256', None)] hash_list.extend([x[:64] for x in submission.get('errors', [])]) hash_list.extend([x[:64] for x in submission.get('results', [])]) if sha256 not in hash_list: return make_api_response({}, f"File {sha256} is not associated to submission {sid}.", 403) if Classification.is_accessible(user['classification'], submission['classification']): submission_meta.update(unflatten(submission['metadata'])) if Classification.enforce: submission_classification = submission.get('classification', file_obj['classification']) submission_meta['classification'] = Classification.max_classification(submission_classification, file_obj['classification']) encoding = request.args.get('encoding', params['download_encoding']) if encoding not in ['raw', 'cart']: return make_api_response({}, f"{encoding.upper()} is not in the valid encoding types: [raw, cart]", 403) if encoding == "raw" and not ALLOW_RAW_DOWNLOADS: return make_api_response({}, "RAW file download has been disabled by administrators.", 403) _, download_path = tempfile.mkstemp() try: with forge.get_filestore() as f_transport: downloaded_from = f_transport.download(sha256, download_path) if not downloaded_from: return make_api_response({}, "The file was not found in the system.", 404) if encoding == 'raw': target_path = download_path else: target_path, name = encode_file(download_path, name, submission_meta) try: return stream_file_response(open(target_path, 'rb'), name, os.path.getsize(target_path)) finally: if target_path: if os.path.exists(target_path): os.unlink(target_path) finally: if download_path: if os.path.exists(download_path): os.unlink(download_path) else: return make_api_response({}, "You are not allowed to download this file.", 403)
def who_am_i(**kwargs): """ Return the currently logged in user as well as the system configuration Variables: None Arguments: None Data Block: None Result example: { "agrees_with_tos": None, # Date the user agreed with TOS "avatar": "data:image/jpg...", # Avatar data block "c12nDef": {}, # Classification definition block "classification": "TLP:W", # Classification of the user "configuration": { # Configuration block "auth": { # Authentication Configuration "allow_2fa": True, # Is 2fa Allowed for the user "allow_apikeys": True, # Are APIKeys allowed for the user "allow_security_tokens": True, # Are Security tokens allowed for the user }, "ui": { # UI Configuration "allow_url_submissions": True, # Are URL submissions allowed "read_only": False, # Is the interface to be displayed in read-only mode "tos": True, # Are terms of service set in the system "tos_lockout": False, # Will agreeing to TOS lockout the user "tos_lockout_notify": False # Will admin be auto-notified when a user is locked out } }, "email": "*****@*****.**", # Email of the user "groups": ["USERS"], # Groups the user if member of "indexes": {}, # Search indexes definitions "is_active": True, # Is the user active "name": "Basic user", # Name of the user "type": ["user", "admin"], # Roles the user is member of "uname": "sgaron-cyber" # Username of the current user } """ user_data = {k: v for k, v in kwargs['user'].items() if k in [ "agrees_with_tos", "classification", "email", "groups", "is_active", "name", "type", "uname"]} user_data['avatar'] = STORAGE.user_avatar.get(kwargs['user']['uname']) user_data['username'] = user_data.pop('uname') user_data['is_admin'] = "admin" in user_data['type'] user_data['roles'] = user_data.pop('type') # System configuration user_data['c12nDef'] = classification_definition user_data['configuration'] = { "auth": { "allow_2fa": config.auth.allow_2fa, "allow_apikeys": config.auth.allow_apikeys, "allow_security_tokens": config.auth.allow_security_tokens, }, "ui": { "allow_url_submissions": config.ui.allow_url_submissions, "read_only": config.ui.read_only, "tos": config.ui.tos not in [None, ""], "tos_lockout": config.ui.tos_lockout, "tos_lockout_notify": config.ui.tos_lockout_notify not in [None, []] }, } user_data['indexes'] = list_all_fields() user_data['settings'] = load_user_settings(kwargs['user']) return make_api_response(user_data)