def test_bad_cert(): """Make sure that the client detects that the test cert is self signed.""" with mocks.Server() as server: try: assemblyline_client.get_client(server.address) assert False except assemblyline_client.ClientError as ce: assert 'CERTIFICATE_VERIFY_FAILED' in str( ce) or 'certificate verify failed' in str(ce)
def test_encrypt_apikey_auth(): """Send an encryped apikey and decrypt it.""" with mocks.Server() as server: assemblyline_client.get_client(server.address, verify=False, apikey=('username', 'ANAPIKEY')) assert len(server.logins) == 1 assert server.logins[0]['user'] == 'username' assert server.logins[0]['apikey'] != 'ANAPIKEY' assert server.private_key.decrypt( b64decode(server.logins[0]['apikey']), 'ERROR') == b'ANAPIKEY'
def test_encrypt_password_auth(): """Send an encryped password and decrypt it.""" with mocks.Server() as server: assemblyline_client.get_client(server.address, verify=False, auth=('username', 'password')) assert len(server.logins) == 1 assert server.logins[0]['user'] == 'username' assert server.logins[0]['password'] != 'password' assert server.private_key.decrypt( b64decode(server.logins[0]['password']), 'ERROR') == b'password'
def main(url: str, username: str, apikey: str, min_score: int, incident_num: int): """ Example: python3 results-analyzer.py --url="https://<domain-of-Assemblyline-instance>" --username="******" --apikey="<api-key-name>:<key>" --incident_num=123 """ # Phase 1: Parameter validation try: validate_parameters(url) except Exception as e: # If there are any exceptions raised at this point, bail! print(e) log.error(e) return # Phase 2: Create the Assemblyline Client al_client = get_client(url, apikey=(username, apikey)) # Phase 3: Open important files and read their contents report_file = open(REPORT_FILE, "a") report_file.write("FilePath,SHA256,Score,URL,Errors\n") # Phase 4: Get submission details for each ingest_id log.debug( f"Searching for the submission for incident number {incident_num}") submission_res = al_client.search.stream.submission( f"params.description:'Incident Number\: {incident_num}' AND max_score:>={min_score}" ) for submission in submission_res: # Phase 5: Wait until the submission has completed state = submission["state"] while state != "completed": msg = f"{submission['sid']} is still in the state:{state}. Sleeping for 2 seconds and trying again." print(msg) log.debug(msg) sleep(2) specific_submission_res = al_client.submission.full( submission["sid"]) state = specific_submission_res["state"] # Deep dive into the submission to get the files full_sub = al_client.submission.full(submission["sid"]) for file in full_sub["files"]: # Report accordingly. msg = f"{full_sub['metadata']['filename']},{file['sha256']},{full_sub['max_score']},{url}/submission/report/{submission['sid']},{full_sub['errors']}\n" print(msg) log.debug(msg) report_file.write(msg) msg = "All done!" print(msg) log.debug(msg)
def client(datastore): user = datastore.user.get('admin') random_pass = get_random_password(length=48) key_name = "key_%s" % get_random_id().lower() user.apikeys[key_name] = { "password": bcrypt.hash(random_pass), "acl": ["R", "W"] } datastore.user.save('admin', user) api_key = "%s:%s" % (key_name, random_pass) c = get_client(UI_HOST, apikey=('admin', api_key), verify=False) return c
def run(self): if self.data_type not in ['file']: self.notSupported() al_client = get_client(self.assemblyline_server, apikey=(self.assemblyline_user, self.assemblyline_key), verify=self.assemblyline_verifyssl) al_client.submit('/path/to/my/file.txt') time.sleep(self.getParam("config.delay", 60)) self.report({'data': self.getData(), 'input': self._input})
def do_source_update(self, service: Service) -> None: self.log.info(f"Connecting to Assemblyline API: {UI_SERVER}...") run_time = time.time() username = self.ensure_service_account() with temporary_api_key(self.datastore, username) as api_key: with tempfile.TemporaryDirectory() as update_dir: al_client = get_client(UI_SERVER, apikey=(username, api_key), verify=False) old_update_time = self.get_source_update_time() self.log.info("Connected!") # Parse updater configuration previous_hashes: dict[str, dict[str, str]] = self.get_source_extra() sources: dict[str, UpdateSource] = {_s['name']: _s for _s in service.update_config.sources} files_sha256: dict[str, dict[str, str]] = {} # Go through each source and download file for source_name, source_obj in sources.items(): source = source_obj.as_primitives() uri: str = source['uri'] default_classification = source.get('default_classification', classification.UNRESTRICTED) try: # Pull sources from external locations (method depends on the URL) files = git_clone_repo(source, old_update_time, self.default_pattern, self.log, update_dir) \ if uri.endswith('.git') else url_download(source, old_update_time, self.log, update_dir) # Add to collection of sources for caching purposes self.log.info(f"Found new {self.updater_type} rule files to process for {source_name}!") validated_files = list() for file, sha256 in files: files_sha256.setdefault(source_name, {}) if previous_hashes.get(source_name, {}).get(file, None) != sha256 and self.is_valid(file): files_sha256[source_name][file] = sha256 validated_files.append((file, sha256)) # Import into Assemblyline self.import_update(validated_files, al_client, source_name, default_classification) except SkipSource: # This source hasn't changed, no need to re-import into Assemblyline self.log.info(f'No new {self.updater_type} rule files to process for {source_name}') if source_name in previous_hashes: files_sha256[source_name] = previous_hashes[source_name] continue self.set_source_update_time(run_time) self.set_source_extra(files_sha256) self.set_active_config_hash(self.config_hash(service)) self.local_update_flag.set()
def test_noauth_submit(mocker): """Submit a file and ensure that the same file is unpacked.""" with mocks.Server() as server: client = assemblyline_client.get_client(server.address, verify=False) submits = server.submits # Submit a file with contents client.submit(path='readme.txt', contents=b'abc123') assert len(submits) == 1 assert b64decode(submits[0]['binary']) == b'abc123' assert submits[0]['name'] == 'readme.txt' submits.pop() # Submit a file from a file mocker.patch('os.path.exists', return_value=True) mocker.patch('assemblyline_client.v3_client.open', mock.mock_open(read_data=b'abc123'), create=True) client.submit(path='readme.txt') assert len(submits) == 1 assert b64decode(submits[0]['binary']) == b'abc123' assert submits[0]['name'] == 'readme.txt' submits.pop()
def _main(arguments): global al_result_to_text signal(SIGINT, SIG_DFL) if sys.platform.startswith("linux"): from signal import SIGPIPE signal(SIGPIPE, SIG_DFL) user = None pw = None cert = None apikey = None transport = "https" host = "localhost" port = 443 kw = {} verify = True config = ConfigParser() config.read([expanduser("~/.al/submit.cfg")]) for section in config.sections(): if section == "auth": if 'user' in config.options('auth'): user = config.get('auth', 'user') if 'password' in config.options('auth'): pw = config.get('auth', 'password') if 'cert' in config.options('auth'): cert = config.get('auth', 'cert') if 'apikey' in config.options('auth'): apikey = config.get('auth', 'apikey') if 'insecure' in config.options('auth'): verify = config.get('auth', 'insecure').lower() not in ['true', 'yes'] elif section == "server": if 'transport' in config.options('server'): transport = config.get('server', 'transport') if 'host' in config.options('server'): host = config.get('server', 'host') if 'port' in config.options('server'): port = config.get('server', 'port') if 'cert' in config.options('server'): verify = config.get('server', 'cert') server = "%s://%s:%s" % (transport, host, port) # parse the command line args from argparse import ArgumentParser parser = ArgumentParser(description=description_string) parser.add_argument('files', metavar='file/dir', nargs='+') parser.add_argument('-v', '--version', action='version', version=__version__) parser.add_argument('-q', '--quiet', action='store_true', help='Runs in quiet mode') parser.add_argument('-a', '--async', dest='async_command', action='store_true', help='Run in asynchronized mode (uses ingest API).') parser.add_argument( '-n', '--no-output', action='store_true', help= 'Only works in conjunction with -a. Ingests the file and does not wait for the output.' ) parser.add_argument( '-i', '--insecure', action='store_true', default=not verify, help= 'Skip server cert validation. DEFAULT: insecure in auth section of ~/.al/submit.cfg' ) parser.add_argument('-t', '--text', action='store_true', help='Dumps results as text instead of json.') parser.add_argument( '-d', '--run-dynamic', action='store_true', help='Adds Dynamic Analysis to the list of service to run.') parser.add_argument( '-u', '--user', default=user, metavar='"user"', help= 'username to be used to connect to AL. DEFAULT: user in auth section of ~/.al/submit.cfg' ) parser.add_argument( '-p', '--password', default=pw, metavar='"MYPASSWORD"', help= 'password of the user. DEFAULT: password in auth section of ~/.al/submit.cfg' ) parser.add_argument('-o', '--output-file', metavar='"/home/user/output.txt"', help='File to write the results to. DEFAULT: stdout') parser.add_argument( '-s', '--server', default=server, metavar='"https://localhost:443"', help='Server to connect to. DEFAULT: transport://host:port in ' 'server section of ~/.al/submit.cfg') parser.add_argument( '-c', '--cert', default=cert, metavar='"/path/to/pki.pem"', help= 'Client cert used to connect to server. DEFAULT: cert in auth section of ~/.al/submit.cfg' ) parser.add_argument( '-k', '--apikey', default=apikey, metavar='"MY_RANDOM_API_KEY"', help= 'apikey to use for the user to login. DEFAULT: apikey in auth section of ~/.al/submit.cfg' ) parser.add_argument('-j', '--json-params', metavar='"{ ... }"', help='A JSON dictionary of submission parameters.') parser.add_argument('-m', '--metadata', metavar='"{ ... }"', help='A JSON dictionary of submission metadata.') parser.add_argument( '--srv-spec', metavar='"{ ... }"', help='A JSON dictionary of service specific parameters.') parser.add_argument( '--server-crt', metavar='"/path/to/server.crt"', help='DEFAULT: cert in server section of ~/.al/submit.cfg') params = parser.parse_args(arguments) args = params.files verbose = not params.quiet async_command = params.async_command no_output = params.no_output json_output = not params.text dynamic = params.run_dynamic user = params.user cert = params.cert pw = params.password apikey = params.apikey if params.insecure: verify = False else: if params.server_crt: verify = params.server_crt if not cert and not user: sys.stderr.write("This server requires authentication...\n") sys.exit(1) if user and not pw and not apikey: if verbose: sys.stderr.write( "You specified a username without a password. What is your password?\n" ) pw = getpass() output = params.output_file if output: f = None try: f = open(output, "ab") except Exception: # pylint: disable=W0702 sys.stderr.write("!!ERROR!! Output file cannot be created (%s)\n" % output) finally: try: f.close() except Exception: # pylint: disable=W0702 pass server = params.server if not server: sys.stderr.write( "!!ERROR!! No server specified, -s option is mandatory.\n\n%s" % parser.format_help()) return -1 if params.metadata: kw['metadata'] = json.loads(params.metadata) if params.json_params: kw["params"] = json.loads(params.json_params) if params.srv_spec: kw.setdefault("params", {}) kw["params"]["service_spec"] = json.loads(params.srv_spec) auth = None api_auth = None if user and apikey: api_auth = (user, apikey) elif user and pw: auth = (user, pw) options = { 'verbose': verbose, 'json_output': json_output, } read_from_pipe = False if sys.platform.startswith("linux") or sys.platform.startswith("freebsd"): try: if select.select([ sys.stdin, ], [], [], 0.0)[0]: read_from_pipe = True except io.UnsupportedOperation: # stdin has probably been replaced with a non-file python object # this is fine. pass if len(args) == 0 and not read_from_pipe: sys.stdout.write("%s\n" % parser.format_help()) return 0 try: client = get_client(server, apikey=api_auth, auth=auth, cert=cert, verify=verify) if isinstance(client, Client4): from assemblyline_client.v4_client.common.submit_utils import al_result_to_text else: from assemblyline_client.v3_client.utils import al_result_to_text except ClientError as e: if e.status_code == 401: sys.stderr.write( "!!ERROR!! Authentication to the server failed.\n") elif e.status_code == 495: sys.stderr.write( "!!ERROR!! Invalid SSL connection to the server:\n\t%s\n" % e) else: raise return 1 if dynamic: p = client.user.submission_params("__CURRENT__") if "Dynamic Analysis" not in p['services']['selected']: p['services']['selected'].append("Dynamic Analysis") if 'params' in kw: p.update(kw['params']) kw['params'] = p if async_command and not no_output: kw['nq'] = "al_submit_%s" % get_random_id() # sanity check path if len(args) == 0 and read_from_pipe: while True: line = sys.stdin.readline() if not line: break line = line.strip() if line == '-': line = '/dev/stdin' if async_command: kw.setdefault('metadata', {}) kw['metadata']['al_submit_id'] = get_id_from_path(line) send_async(client, line, verbose=verbose, **kw) else: send(client, line, output, options, **kw) else: ret_val = 0 file_list = [] for arg in args: if arg == '-': file_list.append('/dev/stdin') elif not exists(arg): sys.stderr.write("!!ERROR!! %s => File does not exist.\n" % arg) ret_val = 1 elif isdir(arg): for root, _, fname_list in walk(arg): for fname in fname_list: file_list.append(join(root, fname)) else: file_list.append(arg) queued_files = [get_id_from_path(f) for f in file_list] output_thread = None if async_command and not no_output: output_thread = start_result_thread(client, queued_files, output, options, **kw) for input_file in file_list: if async_command: kw.setdefault('metadata', {}) kw['metadata']['al_submit_id'] = get_id_from_path(input_file) if not send_async(client, input_file, verbose=verbose, **kw): with ASYNC_LOCK: queued_files.remove(get_id_from_path(input_file)) if verbose: sys.stderr.write( "\tWARNING: Could not send file %s.\n" % input_file) ret_val = 1 else: if not send(client, input_file, output, options, **kw): ret_val = 1 if output_thread: output_thread.join() if ret_val != 0 and len(file_list) > 1: if verbose: sys.stderr.write( "\n** WARNING: al_submit encountered some " "errors while processing multiple files. **\n") return ret_val
def do_local_update(self) -> None: old_update_time = self.get_local_update_time() if not os.path.exists(UPDATER_DIR): os.makedirs(UPDATER_DIR) _, time_keeper = tempfile.mkstemp(prefix="time_keeper_", dir=UPDATER_DIR) if self._service.update_config.generates_signatures: output_directory = tempfile.mkdtemp(prefix="update_dir_", dir=UPDATER_DIR) self.log.info("Setup service account.") username = self.ensure_service_account() self.log.info("Create temporary API key.") with temporary_api_key(self.datastore, username) as api_key: self.log.info(f"Connecting to Assemblyline API: {UI_SERVER}") al_client = get_client(UI_SERVER, apikey=(username, api_key), verify=False) # Check if new signatures have been added self.log.info("Check for new signatures.") if al_client.signature.update_available( since=epoch_to_iso(old_update_time) or '', sig_type=self.updater_type)['update_available']: self.log.info("An update is available for download from the datastore") self.log.debug(f"{self.updater_type} update available since {epoch_to_iso(old_update_time) or ''}") extracted_zip = False attempt = 0 # Sometimes a zip file isn't always returned, will affect service's use of signature source. Patience.. while not extracted_zip and attempt < 5: temp_zip_file = os.path.join(output_directory, 'temp.zip') al_client.signature.download( output=temp_zip_file, query=f"type:{self.updater_type} AND (status:NOISY OR status:DEPLOYED)") self.log.debug(f"Downloading update to {temp_zip_file}") if os.path.exists(temp_zip_file) and os.path.getsize(temp_zip_file) > 0: self.log.debug(f"File type ({os.path.getsize(temp_zip_file)}B): {zip_ident(temp_zip_file, 'unknown')}") try: with ZipFile(temp_zip_file, 'r') as zip_f: zip_f.extractall(output_directory) extracted_zip = True self.log.info("Zip extracted.") except BadZipFile: attempt += 1 self.log.warning(f"[{attempt}/5] Bad zip. Trying again after 30s...") time.sleep(30) except Exception as e: self.log.error(f'Problem while extracting signatures to disk: {e}') break os.remove(temp_zip_file) if extracted_zip: self.log.info("New ruleset successfully downloaded and ready to use") self.serve_directory(output_directory, time_keeper) else: self.log.error("Signatures aren't saved to disk.") shutil.rmtree(output_directory, ignore_errors=True) if os.path.exists(time_keeper): os.unlink(time_keeper) else: self.log.info("No signature updates available.") shutil.rmtree(output_directory, ignore_errors=True) if os.path.exists(time_keeper): os.unlink(time_keeper) else: output_directory = self.prepare_output_directory() self.serve_directory(output_directory, time_keeper)
def test_noauth(): """The test server should let us login with no authentication.""" with mocks.Server() as server: assemblyline_client.get_client(server.address, verify=False) assert len(server.logins) == 1
import jwt import os from assemblyline_client import get_client from flask import Flask, render_template, request, redirect TOKEN = None AL_HOST = os.environ.get("AL_HOST", "127.0.0.1.nip.io") AL_USER = os.environ.get("AL_USER", "admin") AL_APIKEY = os.environ.get("AL_APIKEY", "devkey:admin") OBO_HOST = os.environ.get("OBO_HOST", "127.0.0.1.nip.io") CLIENT = get_client(f"https://{AL_HOST}", apikey=(AL_USER, AL_APIKEY), verify=False) ########################## # App settings app = Flask("obo_test") app.config.update( SESSION_COOKIE_SECURE=False, SECRET_KEY="Not so secret is it?!", PREFERRED_URL_SCHEME='https' ) @app.route("/token/", methods=["GET"]) def get_token(): global TOKEN TOKEN = request.values.get("token") return redirect("/")
def sigma_update() -> None: """ Using an update configuration file as an input, which contains a list of sources, download all the file(s). """ # noinspection PyBroadException try: # Load updater configuration update_config = {} if UPDATE_CONFIGURATION_PATH and os.path.exists( UPDATE_CONFIGURATION_PATH): with open(UPDATE_CONFIGURATION_PATH, 'r') as yml_fh: update_config = yaml.safe_load(yml_fh) else: LOGGER.error( f"Update configuration file doesn't exist: {UPDATE_CONFIGURATION_PATH}" ) exit() # Exit if no update sources given if 'sources' not in update_config.keys( ) or not update_config['sources']: LOGGER.error( f"Update configuration does not contain any source to update from" ) exit() # Initialise al_client server = update_config['ui_server'] user = update_config['api_user'] api_key = update_config['api_key'] LOGGER.info(f"Connecting to Assemblyline API: {server}...") al_client = get_client(server, apikey=(user, api_key), verify=False) LOGGER.info(f"Connected!") # Parse updater configuration previous_update = update_config.get('previous_update', None) previous_hash = json.loads( update_config.get('previous_hash', None) or "{}") sources = { source['name']: source for source in update_config['sources'] } files_sha256 = {} source_default_classification = {} # Go through each source and download file for source_name, source in sources.items(): uri: str = source['uri'] source_default_classification[source_name] = source.get( 'default_classification', classification.UNRESTRICTED) if uri.endswith('.git'): files = git_clone_repo(source, previous_update=previous_update) for file, sha256 in files: files_sha256.setdefault(source_name, {}) if previous_hash.get(source_name, {}).get(file, None) != sha256: try: if val_file(file): files_sha256[source_name][file] = sha256 else: LOGGER.warning( f"{file} was not imported due to failed validation" ) except UnsupportedFeature as e: LOGGER.warning(f"{file} | {e}") else: files = url_download(source, previous_update=previous_update) for file, sha256 in files: files_sha256.setdefault(source_name, {}) if previous_hash.get(source_name, {}).get(file, None) != sha256: if val_file(file): files_sha256[source_name][file] = sha256 else: LOGGER.warning( f"{file} was not imported due to failed validation" ) if files_sha256: LOGGER.info("Found new Sigma rule files to process!") sigma_importer = SigmaImporter(al_client, logger=LOGGER) for source, source_val in files_sha256.items(): total_imported = 0 default_classification = source_default_classification[source] for file in source_val.keys(): try: total_imported += sigma_importer.import_file( file, source, default_classification=default_classification) except ValueError: LOGGER.warning( f"{file} failed to import due to a Sigma error") except ComposerError: LOGGER.warning( f"{file} failed to import due to a YAML-parsing error" ) LOGGER.info( f"{total_imported} signatures were imported for source {source}" ) else: LOGGER.info('No new Sigma rule files to process') if al_client.signature.update_available( since=previous_update or '', sig_type='sigma')['update_available']: LOGGER.info( "An update is available for download from the datastore") if not os.path.exists(UPDATE_OUTPUT_PATH): os.makedirs(UPDATE_OUTPUT_PATH) temp_zip_file = os.path.join(UPDATE_OUTPUT_PATH, 'temp.zip') al_client.signature.download( output=temp_zip_file, query="type:sigma AND (status:NOISY OR status:DEPLOYED)") LOGGER.info( f"sigs downloaded in {os.listdir(UPDATE_OUTPUT_PATH)}") if os.path.exists(temp_zip_file): with ZipFile(temp_zip_file, 'r') as zip_f: zip_f.extractall(UPDATE_OUTPUT_PATH) os.remove(temp_zip_file) # Create the response yaml with open(os.path.join(UPDATE_OUTPUT_PATH, 'response.yaml'), 'w') as yml_fh: yaml.safe_dump(dict(hash=json.dumps(files_sha256)), yml_fh) LOGGER.info( f"New ruleset successfully downloaded and ready to use") LOGGER.info(f"Sigma updater completed successfully") except Exception: LOGGER.exception("Updater ended with an exception!")
def main(url: str, username: str, apikey: str, ttl: int, classification: str, service_selection: str, is_test: bool, path: str, fresh: bool, incident_num: int, retries: int): """ Example: python3 file-submitter.py --url="https://<domain-of-Assemblyline-instance>" --username="******" --apikey="<api-key-name>:<key>" --classification="<classification>" --service_selection="<service-name>,<service-name>" --path "/path/to/compromised/directory" --incident_num=123 """ # Phase 1: Parameter validation try: service_selection = validate_parameters(url, service_selection) except Exception as e: # If there are any exceptions raised at this point, bail! print(e) log.error(e) return # Phase 2: Setting the parameters settings = { "ttl": ttl, "ignore_cache": True, "description": f"Incident Number: {incident_num}", "classification": classification, "services": { "selected": service_selection } } # Phase 3: Test mode if is_test: msg = f"The Assemblyline ingest settings you would use are: {settings}" print(msg) log.debug(msg) return # Phase 4: Initialize key variables hash_table = [] number_of_files_ingested = 0 if fresh and os.path.exists(HASH_FILE): os.remove(HASH_FILE) # Phase 5: Script Resumption Logic # If the script somehow crashed or stopped prematurely, then the text file containing # the hashes which have been ingested to Assemblyline will still exist on the host. # Therefore, we will check if that file exists, and if so, then we will grab the last # hash that has been ingested to Assemblyline and use that as our starting point for # the current run. resume_ingestion_sha = None skip = False if os.path.exists(HASH_FILE): # This grabs the last hash in the file. resume_ingestion_sha = check_output(["tail", "-1", HASH_FILE]).decode().strip("\n") # This adds the most recent hash that has been ingested to the hash table, so that # we do not re-ingest it during this run. if resume_ingestion_sha: hash_table.append(resume_ingestion_sha) skip = True # Create file handlers for the two information files we need. hash_file = open(HASH_FILE, "a+") skipped_file = open(SKIPPED_FILE, "a+") # Phase 6: Create the Assemblyline Client al_client = get_client(url, apikey=(username, apikey)) retry_count = 0 # Phase 7: Recursively go through every file in the provided folder and its sub-folders. for root, dir_names, file_names in os.walk(path): for file_name in file_names: file_path = os.path.join(root, file_name) # Retry up until x number of retries while retry_count < retries: # Wrap everything in a try-catch so we become invincible try: file_size = os.path.getsize(file_path) # If the file is not within the file size bounds, we can't upload it if file_size > MAX_FILE_SIZE: msg = f"{file_path} is too big. Size: {file_size} > {MAX_FILE_SIZE}." print(msg) log.debug(msg) continue elif file_size < MIN_FILE_SIZE: msg = f"{file_path} is too small. Size: {file_size} < {MIN_FILE_SIZE}." print(msg) log.debug(msg) continue # Phase 8: Ingestion Logic # Create a sha256 hash using the file contents. sha = get_id_from_data(open(file_path, "rb").read()) # We only care about files that occur after the last sha in the hash file if resume_ingestion_sha and resume_ingestion_sha == sha: skip = False # If we have yet to come up to the file who matches the last submitted sha, continue looking! if skip: continue # If file is in hash table, don't ingest it if sha in hash_table: continue else: hash_table.append(sha) # Phase 9: Ingestion and logging everything # Pre-ingestion logging pre_ingestion_message = f"{file_path} ({sha}) is about to be ingested." print(pre_ingestion_message) log.debug(pre_ingestion_message) # Actual ingestion resp = al_client.ingest(path=file_path, fname=file_name, params=settings, metadata={"filename": file_path}) # Documenting the hash and the ingest_id into the text files number_of_files_ingested += 1 hash_file.write(f"{sha}\n") ingest_id = resp['ingest_id'] # Post ingestion logging post_ingestion_message = f"{file_path} ({sha}) has been ingested with ingest_id {ingest_id}." print(post_ingestion_message) log.debug(post_ingestion_message) # Success, now break! break except Exception as e: print(e) log.error(e) # Logic for skipping files based on number of retries retry_count += 1 if retry_count >= retries: msg = f"{file_path} was skipped due to {e}." print(msg) skipped_file.write(msg) else: sleep(5) msg = "All done!" print(msg) log.debug(msg)
def yara_update(updater_type, update_config_path, update_output_path, download_directory, externals, cur_logger) -> None: """ Using an update configuration file as an input, which contains a list of sources, download all the file(s). """ # noinspection PyBroadException try: # Load updater configuration update_config = {} if update_config_path and os.path.exists(update_config_path): with open(update_config_path, 'r') as yml_fh: update_config = yaml.safe_load(yml_fh) else: cur_logger.error(f"Update configuration file doesn't exist: {update_config_path}") exit() # Exit if no update sources given if 'sources' not in update_config.keys() or not update_config['sources']: cur_logger.error(f"Update configuration does not contain any source to update from") exit() # Initialise al_client server = update_config['ui_server'] user = update_config['api_user'] api_key = update_config['api_key'] cur_logger.info(f"Connecting to Assemblyline API: {server}...") al_client = get_client(server, apikey=(user, api_key), verify=False) cur_logger.info(f"Connected!") # Parse updater configuration previous_update = update_config.get('previous_update', None) previous_hash = json.loads(update_config.get('previous_hash', None) or "{}") sources = {source['name']: source for source in update_config['sources']} files_sha256 = {} files_default_classification = {} # Create working directory updater_working_dir = os.path.join(tempfile.gettempdir(), 'updater_working_dir') if os.path.exists(updater_working_dir): shutil.rmtree(updater_working_dir) os.makedirs(updater_working_dir) # Go through each source and download file for source_name, source in sources.items(): os.makedirs(os.path.join(updater_working_dir, source_name)) # 1. Download signatures cur_logger.info(f"Downloading files from: {source['uri']}") uri: str = source['uri'] if uri.endswith('.git'): files = git_clone_repo(download_directory, source, cur_logger, previous_update=previous_update) else: files = [url_download(download_directory, source, cur_logger, previous_update=previous_update)] processed_files = set() # 2. Aggregate files file_name = os.path.join(updater_working_dir, f"{source_name}.yar") mode = "w" for file in files: # File has already been processed before, skip it to avoid duplication of rules if file in processed_files: continue cur_logger.info(f"Processing file: {file}") file_dirname = os.path.dirname(file) processed_files.add(os.path.normpath(file)) with open(file, 'r') as f: f_lines = f.readlines() temp_lines = [] for i, f_line in enumerate(f_lines): if f_line.startswith("include"): lines, processed_files = replace_include(f_line, file_dirname, processed_files, cur_logger) temp_lines.extend(lines) else: temp_lines.append(f_line) # guess the type of files that we have in the current file guessed_category = guess_category(file) parser = Plyara() signatures = parser.parse_string("\n".join(temp_lines)) # Ignore "cuckoo" rules if "cuckoo" in parser.imports: parser.imports.remove("cuckoo") # Guess category if guessed_category: for s in signatures: if 'metadata' not in s: s['metadata'] = [] # Do not override category with guessed category if it already exists for meta in s['metadata']: if 'category' in meta: continue s['metadata'].append({'category': guessed_category}) s['metadata'].append({guessed_category: s.get('rule_name')}) # Save all rules from source into single file with open(file_name, mode) as f: for s in signatures: # Fix imports and remove cuckoo s['imports'] = utils.detect_imports(s) if "cuckoo" not in s['imports']: f.write(utils.rebuild_yara_rule(s)) if mode == "w": mode = "a" # Check if the file is the same as the last run if os.path.exists(file_name): cache_name = os.path.basename(file_name) sha256 = get_sha256_for_file(file_name) if sha256 != previous_hash.get(cache_name, None): files_sha256[cache_name] = sha256 files_default_classification[cache_name] = source.get('default_classification', classification.UNRESTRICTED) else: cur_logger.info(f'File {cache_name} has not changed since last run. Skipping it...') if files_sha256: cur_logger.info(f"Found new {updater_type.upper()} rules files to process!") yara_importer = YaraImporter(updater_type, al_client, logger=cur_logger) # Validating and importing the different signatures for base_file in files_sha256: cur_logger.info(f"Validating output file: {base_file}") cur_file = os.path.join(updater_working_dir, base_file) source_name = os.path.splitext(os.path.basename(cur_file))[0] default_classification = files_default_classification.get(base_file, classification.UNRESTRICTED) try: _compile_rules(cur_file, externals, cur_logger) yara_importer.import_file(cur_file, source_name, default_classification=default_classification) except Exception as e: raise e else: cur_logger.info(f'No new {updater_type.upper()} rules files to process...') # Check if new signatures have been added if al_client.signature.update_available(since=previous_update or '', sig_type=updater_type)['update_available']: cur_logger.info("An update is available for download from the datastore") if not os.path.exists(update_output_path): os.makedirs(update_output_path) temp_zip_file = os.path.join(update_output_path, 'temp.zip') al_client.signature.download(output=temp_zip_file, query=f"type:{updater_type} AND (status:NOISY OR status:DEPLOYED)") if os.path.exists(temp_zip_file): with ZipFile(temp_zip_file, 'r') as zip_f: zip_f.extractall(update_output_path) os.remove(temp_zip_file) # Create the response yaml with open(os.path.join(update_output_path, 'response.yaml'), 'w') as yml_fh: yaml.safe_dump(dict(hash=json.dumps(files_sha256)), yml_fh) cur_logger.info(f"New ruleset successfully downloaded and ready to use") cur_logger.info(f"{updater_type.upper()} updater completed successfully") except Exception: cur_logger.exception("Updater ended with an exception!")