def __init__(self, logger=None): if not logger: from assemblyline.common import log as al_log al_log.init_logging('suricata_importer') logger = logging.getLogger('assemblyline.suricata_importer') logger.setLevel(logging.INFO) self.ds = forge.get_datastore() self.classification = forge.get_classification() self.log = logger
def __init__(self, al_client, logger=None): if not logger: from assemblyline.common import log as al_log al_log.init_logging('sigma_importer') logger = logging.getLogger('assemblyline.sigma_importer') logger.setLevel(logging.INFO) self.update_client = al_client self.classification = forge.get_classification() self.log = logger
def __init__(self, importer_type, al_client, logger=None): if not logger: from assemblyline.common import log as al_log al_log.init_logging('yara_importer') logger = logging.getLogger('assemblyline.yara_importer') logger.setLevel(logging.INFO) self.importer_type = importer_type self.update_client = al_client self.parser = Plyara() self.classification = forge.get_classification() self.log = logger
def __init__(self, logger: logging.Logger = None, shutdown_timeout: float = None, config: Config = None, datastore: AssemblylineDatastore = None, redis: RedisType = None, redis_persist: RedisType = None, default_pattern=".*"): self.updater_type = os.environ['SERVICE_PATH'].split('.')[-1].lower() self.default_pattern = default_pattern if not logger: al_log.init_logging(f'updater.{self.updater_type}', log_level=os.environ.get('LOG_LEVEL', "WARNING")) logger = logging.getLogger(f'assemblyline.updater.{self.updater_type}') super().__init__(f'assemblyline.{SERVICE_NAME}_updater', logger=logger, shutdown_timeout=shutdown_timeout, config=config, datastore=datastore, redis=redis, redis_persist=redis_persist) self.update_data_hash = Hash(f'service-updates-{SERVICE_NAME}', self.redis_persist) self._update_dir = None self._update_tar = None self._time_keeper = None self._service: Optional[Service] = None self.event_sender = EventSender('changes.services', host=self.config.core.redis.nonpersistent.host, port=self.config.core.redis.nonpersistent.port) self.service_change_watcher = EventWatcher(self.redis, deserializer=ServiceChange.deserialize) self.service_change_watcher.register(f'changes.services.{SERVICE_NAME}', self._handle_service_change_event) self.signature_change_watcher = EventWatcher(self.redis, deserializer=SignatureChange.deserialize) self.signature_change_watcher.register(f'changes.signatures.{SERVICE_NAME.lower()}', self._handle_signature_change_event) # A event flag that gets set when an update should be run for # reasons other than it being the regular interval (eg, change in signatures) self.source_update_flag = threading.Event() self.local_update_flag = threading.Event() self.local_update_start = threading.Event() # Load threads self._internal_server = None self.expected_threads = { 'Sync Service Settings': self._sync_settings, 'Outward HTTP Server': self._run_http, 'Internal HTTP Server': self._run_internal_http, 'Run source updates': self._run_source_updates, 'Run local updates': self._run_local_updates, } # Only used by updater with 'generates_signatures: false' self.latest_updates_dir = os.path.join(UPDATER_DIR, 'latest_updates') if not os.path.exists(self.latest_updates_dir): os.makedirs(self.latest_updates_dir)
def __init__(self, externals=None, logger=None): if not logger: from assemblyline.common import log as al_log al_log.init_logging('YaraValidator') logger = logging.getLogger('assemblyline.yara_validator') logger.setLevel(logging.WARNING) if not externals: externals = {'dummy': ''} self.log = logger self.externals = externals self.rulestart = re.compile( r'^(?:global )?(?:private )?(?:private )?rule ', re.MULTILINE) self.rulename = re.compile('rule ([^{^:]+)')
def __init__(self, task: ServiceTask): # Initialize logging al_log.init_logging(f'{task.service_name.lower()}', log_level=logging.INFO) self.log = logging.getLogger( f'assemblyline.service.{task.service_name.lower()}') tags = {} for t in task.tags: tags.setdefault(t.type, []) tags[t.type].append(t.value) self._classification: Classification = forge.get_classification() self._service_completed: Optional[str] = None self._service_started: Optional[str] = None self._working_directory: Optional[str] = None self.deep_scan = task.deep_scan self.depth = task.depth self.drop_file: bool = False self.error_message: Optional[str] = None self.error_status: Optional[str] = None self.error_type: str = 'EXCEPTION' self.extracted: List[Dict[str, str]] = [] self.file_name = task.filename self.file_type = task.fileinfo.type self.file_size = task.fileinfo.size self.ignore_filtering = task.ignore_filtering self.min_classification = task.min_classification.value self.max_extracted = task.max_files self.metadata = task.metadata self.md5: str = task.fileinfo.md5 self.mime: str = task.fileinfo.mime or None self.result: Optional[Result] = None self.safelist_config: Dict[str, Any] = task.safelist_config self.service_config: Dict[str, Any] = dict(task.service_config) self.service_context: Optional[str] = None self.service_debug_info: Optional[str] = None self.service_default_result_classification = None self.service_name: str = task.service_name self.service_tool_version: Optional[str] = None self.service_version: Optional[str] = None self.sha1: str = task.fileinfo.sha1 self.sha256: str = task.fileinfo.sha256 self.sid: str = task.sid self.supplementary: List[Dict[str, str]] = [] self.tags = tags self.temp_submission_data: Dict[str, Any] = { row.name: row.value for row in task.temporary_submission_data } self.type: str = task.fileinfo.type
def __init__(self, task: Task) -> None: # Initialize logging for the service al_log.init_logging(f'{task.service_name}', log_level=logging.INFO) self.log = logging.getLogger(f'assemblyline.service.{task.service_name.lower()}') self._working_directory = None self.deep_scan = task.deep_scan self.extracted = task.extracted self.file_name = task.file_name self.file_type = task.file_type self.max_extracted = task.max_extracted self.md5 = task.md5 self.sha1 = task.sha1 self.sha256 = task.sha256 self.sid = task.sid self.task = task
def __init__(self, config: Optional[Dict] = None) -> None: # Load the service attributes from the service manifest self.service_attributes = helper.get_service_attributes() # Start with default service parameters and override with anything provided self.config = self.service_attributes.config if config: self.config.update(config) # Initialize logging for the service log.init_logging(f'{self.service_attributes.name}', log_level=logging.INFO) self.log = logging.getLogger(f'assemblyline.service.{self.service_attributes.name.lower()}') self._task = None self._working_directory = None
def __init__(self, component_name: str, logger: logging.Logger = None, shutdown_timeout: float = SHUTDOWN_SECONDS_LIMIT, config=None): super().__init__(name=component_name) al_log.init_logging(component_name) self.config = config or forge.get_config() self.running = None self.log = logger or logging.getLogger(component_name) self._exception = None self._traceback = None self._shutdown_timeout = shutdown_timeout if shutdown_timeout is not None else SHUTDOWN_SECONDS_LIMIT self._old_sigint = None self._old_sigterm = None self._stopped = False self._last_heartbeat = 0
def run_task_handler(): al_log.init_logging("assemblyline.service.process_handler", log_level=LOG_LEVEL) log = logging.getLogger("assemblyline.service.process_handler") # Start the two processes rs_p = Popen(['python3', '-m', 'assemblyline_v4_service.run_service']) th_p = Popen(['python3', '-m', 'assemblyline_service_client.task_handler']) def forward_signal(signal_number, _frame): th_p.send_signal(signal_number) signal.signal(signal.SIGUSR1, forward_signal) signal.signal(signal.SIGUSR2, forward_signal) signal.signal(signal.SIGTERM, forward_signal) while True: check_processes(rs_p, th_p, log) # Wait 2 seconds before polling process status time.sleep(2)
def __init__(self, config: Optional[Dict] = None) -> None: # Load the service attributes from the service manifest self.service_attributes = helper.get_service_attributes() # Start with default service parameters and override with anything provided self.config = self.service_attributes.config if config: self.config.update(config) self.name = self.service_attributes.name.lower() # Initialize logging for the service log.init_logging(f'{self.service_attributes.name}', log_level=LOG_LEVEL) self.log = logging.getLogger(f'assemblyline.service.{self.name}') # Replace warning/error methods with our own patched version self._log_warning = self.log.warning self._log_error = self.log.error self.log.warning = self._warning self.log.error = self._error self._task = None self._working_directory = None # Initialize interface for interacting with system safelist self._api_interface = None self.dependencies = self._get_dependencies_info() self.ontologies: Dict = None # Updater-related self.rules_directory: str = None self.rules_list: list = [] self.update_time: int = None self.rules_hash: str = None
from elasticapm.contrib.flask import ElasticAPM from flask import Flask from flask.logging import default_handler from assemblyline.common import forge, log as al_log from assemblyline_service_server.api.v1.file import file_api from assemblyline_service_server.api.v1.service import service_api from assemblyline_service_server.api.v1.task import task_api from assemblyline_service_server.api.v1.safelist import safelist_api from assemblyline_service_server.healthz import healthz config = forge.get_config() # Prepare the logger al_log.init_logging('svc') LOGGER = logging.getLogger('assemblyline.svc_server') LOGGER.info("Service server ready to receive connections...") # Prepare the app app = Flask('svc-server') app.config['SECRET_KEY'] = config.ui.secret_key app.register_blueprint(healthz) app.register_blueprint(file_api) app.register_blueprint(service_api) app.register_blueprint(task_api) app.register_blueprint(safelist_api) # Setup logging app.logger.setLevel(LOGGER.getEffectiveLevel())
import logging import os from flask import Flask from flask_socketio import SocketIO from assemblyline_ui.sio.alert import AlertMonitoringNamespace from assemblyline_ui.sio.live_submission import LiveSubmissionNamespace from assemblyline_ui.sio.status import SystemStatusNamespace from assemblyline_ui.sio.submission import SubmissionMonitoringNamespace from assemblyline.common import forge, log as al_log config = forge.get_config() # Prepare the logger al_log.init_logging("ui") LOGGER = logging.getLogger('assemblyline.ui.socketio') LOGGER.info("SocketIO server ready to receive connections...") # Prepare the app app = Flask('socketio') app.config['SECRET_KEY'] = config.ui.secret_key # If the environment says we should prefix our app by something, do so if 'APPLICATION_ROOT' in os.environ: LOGGER.info( f"Flask application root changing: {os.environ['APPLICATION_ROOT']}") app.config['APPLICATION_ROOT'] = os.environ['APPLICATION_ROOT'] app.config['SESSION_COOKIE_PATH'] = '/' # NOTE: we need to run in threading mode while debugging otherwise, use gevent
from __future__ import annotations import json import logging from typing import List, Union, Optional, Dict, Any from assemblyline.common import forge from assemblyline.common import log as al_log from assemblyline.common.attack_map import attack_map, software_map, group_map, revoke_map from assemblyline.common.dict_utils import unflatten from assemblyline.common.str_utils import StringTable, safe_str from assemblyline_v4_service.common.helper import get_service_attributes, get_heuristics al_log.init_logging('service.result') log = logging.getLogger('assemblyline.service.result') Classification = forge.get_classification() SERVICE_ATTRIBUTES = get_service_attributes() BODY_FORMAT = StringTable( 'BODY_FORMAT', [ ('TEXT', 0), ('MEMORY_DUMP', 1), ('GRAPH_DATA', 2), ('URL', 3), ('JSON', 4), ('KEY_VALUE', 5), ('PROCESS_TREE', 6), ('TABLE', 7), ('IMAGE', 8),
if th_rc is not None: log.info(f"task_handler: exit({th_rc})") service_process.terminate() exit(th_rc) # If the service process has crashed tell the task handler something is wrong, # then wait for it to exit voluntarily if rs_rc is not None: log.error( f"The service has crashed with exit code: {rs_rc}. The container will be stopped..." ) task_handler_process.send_signal(signal.SIGUSR1) if __name__ == '__main__': al_log.init_logging("assemblyline.service.process_handler", log_level=LOG_LEVEL) log = logging.getLogger("assemblyline.service.process_handler") # Start the two processes rs_p = Popen(['python3', '-m', 'assemblyline_v4_service.run_service']) th_p = Popen(['python3', '-m', 'assemblyline_service_client.task_handler']) def forward_signal(signal_number, _frame): th_p.send_signal(signal_number) signal.signal(signal.SIGUSR1, forward_signal) signal.signal(signal.SIGUSR2, forward_signal) signal.signal(signal.SIGTERM, forward_signal) while True: check_processes(rs_p, th_p)
import logging import os import shutil import time from copy import deepcopy import certifi import requests import yaml from assemblyline.common import log as al_log from assemblyline.common.digests import get_sha256_for_file from assemblyline.common.isotime import iso_to_epoch al_log.init_logging('service_updater') LOGGER = logging.getLogger('assemblyline.updater.service') UPDATE_CONFIGURATION_PATH = os.environ.get('UPDATE_CONFIGURATION_PATH', None) UPDATE_OUTPUT_PATH = os.environ.get('UPDATE_OUTPUT_PATH', "/tmp/updater_output") def test_file(_): return True def url_update(test_func=test_file) -> None: """ Using an update configuration file as an input, which contains a list of sources, download all the file(s) which
DEBUG = config.ui.debug VERSION = os.environ.get( 'ASSEMBLYLINE_VERSION', f"{FRAMEWORK_VERSION}.{SYSTEM_VERSION}.{BUILD_MINOR}.dev0") AUTH_KEY = os.environ.get('SERVICE_API_KEY', 'ThisIsARandomAuthKey...ChangeMe!') RATE_LIMITER = Counters(prefix="quota", host=redis, track_counters=True) # End of Configuration ################################################################# ################################################################# # Prepare loggers config.logging.log_to_console = config.logging.log_to_console or DEBUG al_log.init_logging('svc', config=config) LOGGER = logging.getLogger('assemblyline.svc') LOGGER.debug('Logger ready!') # End of prepare logger ################################################################# ################################################################# # Global instances STORAGE = forge.get_datastore(config=config) FILESTORE = forge.get_filestore(config=config) LOCK = threading.Lock() TASKING_CLIENT = TaskingClient(datastore=STORAGE,
if th_rc is not None: log.info(f"task_handler: exit({th_rc})") service_process.terminate() exit(th_rc) # If the service process has crashed tell the task handler something is wrong, # then wait for it to exit voluntarily if rs_rc is not None: log.error( f"The service has crashed with exit code: {rs_rc}. The container will be stopped..." ) task_handler_process.send_signal(signal.SIGUSR1) if __name__ == '__main__': al_log.init_logging("assemblyline.service.process_handler") log = logging.getLogger("assemblyline.service.process_handler") # Start the two processes rs_p = Popen(['python3', '-m', 'assemblyline_v4_service.run_service']) th_p = Popen(['python3', '-m', 'assemblyline_service_client.task_handler']) def forward_signal(signal_number, _frame): th_p.send_signal(signal_number) signal.signal(signal.SIGUSR1, forward_signal) signal.signal(signal.SIGUSR2, forward_signal) signal.signal(signal.SIGTERM, forward_signal) while True: check_processes(rs_p, th_p)
from yaml.composer import ComposerError from pysigma.exceptions import UnsupportedFeature import certifi import requests import yaml from assemblyline_client import get_client from git import Repo from assemblyline.common import log as al_log, forge from assemblyline.common.digests import get_sha256_for_file from assemblyline.common.isotime import iso_to_epoch from sigma_importer import SigmaImporter from pysigma.pysigma import val_file al_log.init_logging('updater.sigma') classification = forge.get_classification() LOGGER = logging.getLogger('assemblyline.updater.sigma') UPDATE_CONFIGURATION_PATH = os.environ.get('UPDATE_CONFIGURATION_PATH', "/tmp/sigma_updater_config.yaml") UPDATE_OUTPUT_PATH = os.environ.get('UPDATE_OUTPUT_PATH', "/tmp/sigma_updater_output") UPDATE_DIR = os.path.join(tempfile.gettempdir(), 'sigma_updates') def add_cacert(cert: str): # Add certificate to requests cafile = certifi.where() with open(cafile, 'a') as ca_editor:
import yaml from tempfile import gettempdir from assemblyline.common import forge, log as al_log from assemblyline.common.backupmanager import DistributedBackup from assemblyline.common.security import get_totp_token, generate_random_secret from assemblyline.common.uid import get_random_id from assemblyline.odm.models.signature import RULE_STATUSES from assemblyline.remote.datatypes.hash import Hash warnings.filterwarnings("ignore") config = forge.get_config() config.logging.log_to_console = False al_log.init_logging('cli') PROCESSES_COUNT = 50 COUNT_INCREMENT = 500 DATASTORE = None t_count = 0 t_last = time.time() class NullLogger(object): @staticmethod def info(msg, *args, **kwargs): pass @staticmethod def warning(msg, *args, **kwargs):
def core(request, redis, filestore, config): from assemblyline.common import log as al_log al_log.init_logging("simulation") fields = CoreSession() fields.redis = redis fields.ds = ds = forge.get_datastore() fields.config = config forge.config_singletons[False, None] = fields.config threads = [] fields.filestore = filestore threads: List[ServerBase] = [ # Start the ingester components IngesterInput(datastore=ds, redis=redis, persistent_redis=redis), IngesterSubmitter(datastore=ds, redis=redis, persistent_redis=redis), IngesterInternals(datastore=ds, redis=redis, persistent_redis=redis), # Start the dispatcher FileDispatchServer(datastore=ds, redis=redis, redis_persist=redis), SubmissionDispatchServer(datastore=ds, redis=redis, redis_persist=redis), # Start plumber Plumber(datastore=ds, redis=redis, redis_persist=redis, delay=0.5), ] stages = get_service_stage_hash(redis) ingester_input_thread: IngesterInput = threads[0] fields.ingest = ingester_input_thread fields.ingest_queue = ingester_input_thread.ingester.ingest_queue ds.ds.service = MockCollection(Service) ds.ds.service_delta = MockCollection(Service) ds.service.save('pre_0', dummy_service('pre', 'EXTRACT')) ds.service_delta.save('pre', dummy_service('pre', 'EXTRACT')) stages.set('pre', ServiceStage.Running) threads.append(MockService('pre', ds, redis, filestore)) fields.pre_service = threads[-1] ds.service.save('core-a_0', dummy_service('core-a', 'CORE')) ds.service_delta.save('core-a', dummy_service('core-a', 'CORE')) stages.set('core-a', ServiceStage.Running) threads.append(MockService('core-a', ds, redis, filestore)) ds.service.save('core-b_0', dummy_service('core-b', 'CORE')) ds.service_delta.save('core-b', dummy_service('core-b', 'CORE')) threads.append(MockService('core-b', ds, redis, filestore)) stages.set('core-b', ServiceStage.Running) ds.service.save('finish_0', dummy_service('finish', 'POST')) ds.service_delta.save('finish', dummy_service('finish', 'POST')) threads.append(MockService('finish', ds, redis, filestore)) stages.set('finish', ServiceStage.Running) for t in threads: t.daemon = True t.start() def stop_core(): [tr.close() for tr in threads] [tr.stop() for tr in threads] [tr.raising_join() for tr in threads] request.addfinalizer(stop_core) return fields
def get_signup_queue(key): return ExpiringSet(f"signup_id_{key}", host=config.core.redis.nonpersistent.host, port=config.core.redis.nonpersistent.port, ttl=60 * 15) # End of Configuration ################################################################# ################################################################# # Prepare loggers config.logging.log_to_console = config.logging.log_to_console or DEBUG al_log.init_logging("ui", config=config) AUDIT_KW_TARGET = [ "sid", "sha256", "copy_sid", "filter", "query", "username", "group", "rev", "wq_id", "bucket", "cache_key", "alert_key", "alert_id", "url", "q", "fq", "file_hash", "heuristic_id", "error_key", "mac", "vm_type", "vm_name", "config_name", "servicename", "vm" ] AUDIT_LOG = logging.getLogger('assemblyline.ui.audit') LOGGER = logging.getLogger('assemblyline.ui') if AUDIT: AUDIT_LOG.setLevel(logging.INFO) if DEBUG:
cur_logger.info("An update is available for download from the datastore") if not os.path.exists(update_output_path): os.makedirs(update_output_path) temp_zip_file = os.path.join(update_output_path, 'temp.zip') al_client.signature.download(output=temp_zip_file, query=f"type:{updater_type} AND (status:NOISY OR status:DEPLOYED)") if os.path.exists(temp_zip_file): with ZipFile(temp_zip_file, 'r') as zip_f: zip_f.extractall(update_output_path) os.remove(temp_zip_file) # Create the response yaml with open(os.path.join(update_output_path, 'response.yaml'), 'w') as yml_fh: yaml.safe_dump(dict(hash=json.dumps(files_sha256)), yml_fh) cur_logger.info(f"New ruleset successfully downloaded and ready to use") cur_logger.info(f"{updater_type.upper()} updater completed successfully") except Exception: cur_logger.exception("Updater ended with an exception!") if __name__ == '__main__': al_log.init_logging('updater.yara') logger = logging.getLogger('assemblyline.updater.yara') yara_update("yara", UPDATE_CONFIGURATION_PATH, UPDATE_OUTPUT_PATH, UPDATE_DIR, YARA_EXTERNALS, logger)