def main():
log.init_logging('run_service')
if len(sys.argv) != 2:
usage()
exit(1)
name = sys.argv[1]
try:
svc_class = class_by_name(name) if '.' in name else service_by_name(
name)
except:
print 'Could not load service "%s".\n%s' % (name,
get_valid_service_list())
raise
logger.info('Running service in stand-alone mode. CTRL-C to exit.')
# noinspection PyBroadException
try:
cfg = forge.get_datastore().get_service(svc_class.SERVICE_NAME).get(
"config", {})
except: # pylint: disable=W0702
cfg = {}
service_driver = ServiceDriver(svc_class, cfg, 86400, NUM_WORKERS)
service_driver.start()
try:
while True:
send_minimal_heartbeat(svc_class.SERVICE_NAME, NUM_WORKERS)
time.sleep(config.system.update_interval)
except KeyboardInterrupt:
print 'Exiting.'
finally:
service_driver.stop_hard()
def main(shard):
log.init_logging('dispatcher')
ds = forge.get_datastore()
service_proxies = ServiceProxyManager(ds.list_service_keys())
dispatcher = Dispatcher(service_proxies, shard=shard, debug=False)
dispatcher.start()
def __init__(self, externals=None, logger=None):
if not logger:
from assemblyline.al.common import log as al_log
al_log.init_logging('YaraValidator')
logger = logging.getLogger('assemblyline.yara_validator')
logger.setLevel(logging.WARNING)
if not externals:
externals = {'dummy': ''}
self.log = logger
self.externals = externals
self.rulestart = re.compile(r'^(?:global )?(?:private )?(?:private )?rule ', re.MULTILINE)
self.rulename = re.compile('rule ([^{^:]+)')
def __init__(self, logger=None):
if not logger:
from assemblyline.al.common import log as al_log
al_log.init_logging('yara_importer')
logger = logging.getLogger('assemblyline.yara_importer')
logger.setLevel(logging.INFO)
yara_parser_class = forge.get_yara_parser()
self.ds = forge.get_datastore()
self.yp = yara_parser_class()
self.log = logger
self._id_cache = {}
self._name_cache = []
def main():
log.init_logging('test')
logger = logging.getLogger('assemblyline.test')
store = forge.get_datastore()
sids = []
for x in store.stream_search(
'submission',
'times.completed:[2015-01-30T00:00:00.000Z TO 2015-01-30T00:59:59.999Z]'
):
sid = x['submission.sid']
sids.append(sid)
count = 0
submissions = store.get_submissions(sids)
for submission in submissions:
if submission.get('state', '') != 'completed':
continue
if len(submission['files']) != 1:
continue
_, srl = submission['files'][0]
fileinfo = store.get_file(srl)
if not fileinfo:
continue
submission = submission.get('submission', {})
if not submission:
continue
metadata = submission.get('metadata', {})
if not metadata:
continue
metadata['ignore_submission_cache'] = True
metadata['ignore_cache'] = False
metadata['md5'] = fileinfo['md5']
metadata['sha1'] = fileinfo['sha1']
metadata['sha256'] = fileinfo['sha256']
metadata['size'] = fileinfo['size']
resubmit(metadata)
count += 1
if count >= 1000:
break
logger.info('Resubmitted %d submissions for testing', count)
def main():
log.init_logging('hostagent')
agent = HostAgent()
if len(sys.argv) > 1:
if len(sys.argv) == 2:
if (sys.argv[1]
== '--sysprep') or sys.argv[1] == '--updateandexit':
result = agent.sysprep()
print 'SysPrep: %s' % str(result)
exit(0)
elif sys.argv[1] == '--register':
result = agent.register_host()
print "Registration Result: %s" % str(result)
exit(0)
else:
usage()
exit(1)
else:
usage()
exit(1)
agent.serve_forever()
def subprocess_entry_point(self):
""" The Worker's initial entry point after being spawned."""
# Ignore signal.SIGINT. Let parent handle it and let us know when to stop via self.should_run.
signal.signal(signal.SIGINT, signal.SIG_IGN)
try:
if platform.system() == 'Windows':
# log init doesn't servive the 'fork' on windows. redo.
log.init_logging('hostagent')
# Name our process after the service type. Makes 'ps' listings easier to read.
try_setproctitle(self.service_cls.SERVICE_NAME)
self.log = logging.getLogger('assemblyline.svc.worker.%s' %
self.service_cls.SERVICE_NAME.lower())
msgs = forge.apply_overrides(self.config_overrides)
if msgs:
self.log.info("Using %s.", " and ".join(msgs))
self.ingest_queue = forge.get_service_queue(
self.service_cls.SERVICE_NAME) # remote job queue.
self._stats_sink = forge.get_metrics_sink()
self.log.info("Instantiating supervisor thread")
self._current_work_items_lock = threading.Lock()
self._supervisor_thread = threading.Thread(
name='service-drainer', target=self._supervisor_thread_main)
self._supervisor_thread.start()
self.log.info("Supervisor thread instantiated")
self._run_service_until_shutdown()
self.log.info('_run_service has exited. we must be stopping')
except KeyboardInterrupt:
self.should_run = False
# This should happen if the signal.signal call above is working as expected.
return EXIT_INTERRUPTED
def main(shard):
log.init_logging('dispatcher')
logger = logging.getLogger('assemblyline.dispatch')
r = redis.StrictRedis(config.core.redis.nonpersistent.host,
config.core.redis.nonpersistent.port,
config.core.redis.nonpersistent.db)
r.delete('ingest-queue-' + shard)
store = forge.get_datastore()
store.commit_index('submission')
query = 'state:submitted AND times.submitted:[NOW-1DAY TO *]'
sids = []
for x in store.stream_search('submission', query):
sid = x['submission.sid']
if str(forge.determine_dispatcher(sid)) == shard:
sids.append(sid)
count = 0
submissions = store.get_submissions(sids)
for submission in submissions:
if submission.get('state', '') != 'submitted':
sid = submission.get('sid', '')
if sid:
store.save_submission(sid, submission)
continue
submission['request'] = {}
for path, srl in submission['files']:
submission['fileinfo'] = store.get_file(srl)
submission['request']['path'] = path
submission['request']['srl'] = srl
resubmit(submission)
count += 1
logger.info('Resubmitted %d submissions to dispatcher %s.', count, shard)
class ControllerClient(AgentClient):
def __init__(self, sender=None, async=False):
super(ControllerClient, self).__init__(async, sender)
def heartbeat(self, mac):
return self._send_agent_rpc('Controller.' + mac, ControllerRequest.HEARTBEAT)
# noinspection PyUnusedLocal
def stop(self, mac):
return self._send_agent_rpc('Controller.' + mac, ControllerRequest.STOP)
# noinspection PyUnusedLocal
def start(self, mac):
return self._send_agent_rpc('Controller.' + mac, ControllerRequest.START)
# noinspection PyUnusedLocal
def restart(self, mac):
return self._send_agent_rpc('Controller.' + mac, ControllerRequest.RESTART)
# noinspection PyUnusedLocal
def status(self, mac):
return self._send_agent_rpc('Controller.' + mac, ControllerRequest.STATUS)
if __name__ == '__main__':
al_log.init_logging('controller')
hc = HostController()
hc.serve_forever()
hc.log.info("Controller Terminated")
#!/usr/bin/env python
import logging
from assemblyline.common.isotime import now_as_local
from assemblyline.al.common import forge, log as al_log
config = forge.get_config()
Classification = forge.get_classification()
al_log.init_logging('signature_statistics')
log = logging.getLogger('assemblyline.signature_statistics')
log.info("Generating signature statistics")
store = forge.get_datastore()
output = {"timestamp": None, "stats": None}
stats = {}
sig_list = [(x['meta.id'], x['meta.rule_version'], x['name'],
x.get('meta.classification', Classification.UNRESTRICTED))
for x in store.stream_search(
"signature",
"name:*",
fl="name,meta.id,meta.rule_version,meta.classification")]
for sid, rev, name, classification in sig_list:
key = "%sr.%s" % (sid, rev)
res = store.stats_search("result",
query='result.tags.value:"%s"' % name,
stats_fields=["result.score"])["result.score"]
#!/usr/bin/env python
import logging
from assemblyline.al.common import forge
from assemblyline.al.common import log as al_log
from assemblyline.al.common import queue
config = forge.get_config()
DATABASE_NUM = 4
al_log.init_logging('expiry_worker')
log = logging.getLogger('assemblyline.expiry_worker')
# noinspection PyBroadException
def main():
ds = forge.get_datastore()
fs = forge.get_filestore()
submission_queue = queue.NamedQueue('d-submission', db=DATABASE_NUM)
result_queue = queue.NamedQueue('d-result', db=DATABASE_NUM)
file_queue = queue.NamedQueue('d-file', db=DATABASE_NUM)
error_queue = queue.NamedQueue('d-error', db=DATABASE_NUM)
dynamic_queue = queue.NamedQueue('d-dynamic', db=DATABASE_NUM)
alert_queue = queue.NamedQueue('d-alert', db=DATABASE_NUM)
filescore_queue = queue.NamedQueue('d-filescore', db=DATABASE_NUM)
emptyresult_queue = queue.NamedQueue('d-emptyresult', db=DATABASE_NUM)
log.info("Ready!")
queues = [
submission_queue, result_queue, file_queue, error_queue, dynamic_queue,
#!/usr/bin/env python
import json
import logging
import redis
import time
from assemblyline.common import isotime
from assemblyline.al.common import forge, log as al_log
al_log.init_logging("quota_sniper")
logger = logging.getLogger('assemblyline.quota_sniper')
logger.setLevel(logging.INFO)
config = forge.get_config()
client = redis.StrictRedis(host=config.core.redis.nonpersistent.host,
port=config.core.redis.nonpersistent.port,
db=config.core.redis.nonpersistent.db)
persist = redis.StrictRedis(host=config.core.redis.persistent.host,
port=config.core.redis.persistent.port,
db=config.core.redis.persistent.db)
time_diff = 60 * 5 # Anything older than 5 minutes...
quota_time_diff = 60 * 60 # Anything older than 1 hour...
while True:
# API Quota tracking
data = client.hgetall('c-tracker-quota')
if data:
import os
import time
from threading import Thread
from assemblyline.common.isotime import now_as_iso, now, iso_to_epoch
from assemblyline.al.common import forge, log as al_log, queue
from assemblyline.al.core.datastore import SearchException
DATABASE_NUM = 4
QUERY = "__expiry_ts__:[* TO NOW-12HOUR]" # Delay expiry by 12 hours so we expire peak data during offpeak hours
SLEEP_TIME = 5
MAX_QUEUE_LENGTH = 100000
config = forge.get_config()
al_log.init_logging('expiry')
log = logging.getLogger('assemblyline.expiry')
def load_expired(datastore, name, delete_queue):
# Keep track of the last 60 minutes we've processed and don't reprocess them until they are out of the ignored
# window to let time for the index to be commited to memory
max_rows = 500
max_depth = 5000
ignored_window = []
max_window_size = 60
hosts = datastore.hosts
host_id = 0
log.debug("Expiry will cycle through the following hosts: %s" %
#!/usr/bin/env python
import time
import logging
from assemblyline.al.common import forge, log as al_log, queue
from assemblyline.al.core.datastore import SearchException
from assemblyline.common.charset import safe_str
ds = forge.get_datastore()
DATABASE_NUM = 4
config = forge.get_config()
al_log.init_logging('workflow_filter')
log = logging.getLogger('assemblyline.workflow_filter')
action_queue = queue.PriorityQueue('alert-actions', db=DATABASE_NUM)
QUEUE_PRIORITY = -1
def get_last_reporting_ts(p_start_ts):
log.info(
"Finding reporting timestamp for the last alert since {start_ts}...".
format(start_ts=p_start_ts))
args = [('sort', 'reporting_ts desc'), ('rows', '1'),
('fl', 'reporting_ts')]
result = ds.direct_search(
"alert",
"reporting_ts:[{start_ts} TO *]".format(start_ts=p_start_ts),
args=args)
docs = result.get('response', {}).get('docs', [{}]) or [{}]
ret_ts = docs[0].get("reporting_ts", p_start_ts)
return ret_ts
#!/usr/bin/env python
import os
import subprocess
import logging
from assemblyline.common.net import get_hostname, get_hostip
from assemblyline.al.common import log as al_log, queue
from assemblyline.al.common import message
al_log.init_logging('harddrive_monitor')
log = logging.getLogger('assemblyline.harddrive_monitor')
def is_drive_ok(smart_output):
for line in smart_output.splitlines():
if "SMART Health Status" in line:
status = line.split("SMART Health Status:")[1].strip()
if status == "OK":
return True
else:
return False
elif "SMART overall-health self-assessment test result" in line:
status = line.split(
"SMART overall-health self-assessment test result:")[1].strip(
)
if status == "PASSED":
return True
else:
return False
return False
#!/usr/bin/env python
"""
Alerter
Alerter is responsible for monitoring the alert queue and creating alerts.
"""
import logging
import signal
from assemblyline.common import net
from assemblyline.common.isotime import now
from assemblyline.al.common import forge
from assemblyline.al.common import log
config = forge.get_config()
log.init_logging("alerter")
from assemblyline.al.common import counter
from assemblyline.al.common import queue
persistent_settings = {
'db': config.core.redis.persistent.db,
'host': config.core.redis.persistent.host,
'port': config.core.redis.persistent.port,
}
alertq_name = 'm-alert'
commandq_name = 'a-command'
create_alert = forge.get_create_alert()
datastore = forge.get_datastore()
exit_msgs = ['server closed the connection unexpectedly']
#!/usr/bin/env python
import logging
import os
from assemblyline.common.isotime import now_as_iso, epoch_to_iso, iso_to_epoch
from assemblyline.al.common import forge
from assemblyline.al.common import log
from assemblyline.al.common import queue
config = forge.get_config()
datastore = forge.get_datastore()
log.init_logging('idx_to_journal')
directory = config.core.expiry.journal.directory
emptyresult_queue = queue.NamedQueue(
"ds-emptyresult",
db=config.core.redis.persistent.db,
host=config.core.redis.persistent.host,
port=config.core.redis.persistent.port,
)
logger = logging.getLogger('assemblyline.idx_to_journal')
max_open_files = 1
path_and_filehandle = []
path_to_filehandle = {}
previous = []
def get_filehandle(path):
fh = path_to_filehandle.get(path, None)
#!/usr/bin/env python
import logging
import time
from threading import Thread
from assemblyline.al.common import forge, log as al_log, queue
from al_ui.helper.user import add_access_control
config = forge.get_config()
al_log.init_logging('alert_actions')
log = logging.getLogger('assemblyline.alert_actions')
DATABASE_NUM = 4
DEFAULT_QUEUE_PRIORITY = -2
EXTENDED_SCAN_QUEUE_PRIORITY = 0
TASKER_COUNT = config.core.alert_actions.tasker_count
WORKER_COUNT = config.core.alert_actions.worker_count
def determine_worker_id(event_id):
return int(event_id, 16) % WORKER_COUNT
class AlertAction(object):
def __init__(self):
self.worker_threads = {
x: Thread(target=run_worker_thread,
args=(x, ),
name="worker-%s" % x)
for x in range(WORKER_COUNT)
my_logger.info(output_metrics)
try:
self.es.create(
"al_metrics-%s" % timestamp[:10].replace("-", "."),
component_type, output_metrics)
except Exception as e:
my_logger.exception(e)
my_logger.info("Metrics aggregated... Waiting for next run.")
if __name__ == '__main__':
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
config = forge.get_config()
al_log.init_logging('metricsd')
log = logging.getLogger('assemblyline.metricsd')
elastic_ip = config.get('logging', {}).get('logserver',
{}).get('node', None)
elastic_port = config.get('logging', {}).get('logserver',
{}).get('elastic',
{}).get('port', 9200)
if not elastic_ip or not elastic_port:
log.error(
"Elasticsearch cluster not configured in the seed. There is no need to gather stats on this box."
)
sys.exit(1)
mserver = MetricsServer('SsMetrics', log, elastic_ip, elastic_port)
from assemblyline.al.common import forge, log as al_log, queue
config = forge.get_config()
# Run config
DATABASE_NUM = 3
RETRY_PRINT_THRESHOLD = 1000
PROCESSES_COUNT = 50
COUNT_INCREMENT = 1000
LOW_THRESHOLD = 10000
HIGH_THRESHOLD = 50000
DEBUG = False
DO_SYS_BUCKETS = True
# Logger
al_log.init_logging('reindex')
log = logging.getLogger('assemblyline.reindex')
# Globals
ds = forge.get_datastore()
reindex_queue = queue.NamedQueue('r-index', db=DATABASE_NUM)
done_queue = queue.NamedQueue("r-done", db=DATABASE_NUM)
bucket_error = []
bucket_map = {
"node": ds.nodes,
"profile": ds.profiles,
"signature": ds.signatures,
"user": ds.users,
"alert": ds.alerts,
"file": ds.files,
def __exit__(self, unused1, unused2, unused3):
with self.SCAN_LOCK_LOCK:
l = self.SCAN_LOCK[self.scan_key]
l[0] -= 1
if l[0] == 0:
del self.SCAN_LOCK[self.scan_key]
l[1].release()
Timeout = namedtuple('Timeout', ['time', 'scan_key'])
Classification = forge.get_classification()
config = forge.get_config()
constants = forge.get_constants()
log.init_logging("middleman")
logger = logging.getLogger('assemblyline.middleman')
persistent = {
'db': config.core.redis.persistent.db,
'host': config.core.redis.persistent.host,
'port': config.core.redis.persistent.port,
}
shards = 1
try:
shards = int(config.core.middleman.shards)
except AttributeError:
logger.warning("No shards setting. Defaulting to %d.", shards)
shard = '0'
import elasticsearch
import json
import logging
import psutil
import os
import sys
import urllib2
from apscheduler.scheduler import Scheduler
from assemblyline.common.net import get_hostname, get_hostip
from assemblyline.al.common import forge, log as al_log
from assemblyline.common.isotime import now_as_local, now_as_iso
config = forge.get_config(static_seed=os.getenv("AL_SEED_STATIC"))
al_log.init_logging('system_metrics')
log = logging.getLogger('assemblyline.system_metrics')
previous_net_io = None
previous_disk_io = None
SOLR_ADMIN_URL = '%s/admin/mbeans?stats=true&wt=json&cat=QUERYHANDLER&cat=CORE&key=/update&key=/select&key=searcher'
SOLR_CORE_URL = 'admin/cores?wt=json&indexInfo=false'
SOLR_URL_BUILDER = 'http://localhost:8093/internal_solr/%s'
RIAK_STATS_URL = 'http://localhost:8098/stats'
def calculate_system_metrics(es, cur_ip, cur_host):
global previous_disk_io, previous_net_io
log.info("Starting system metrics calculation...")
#!/usr/bin/env python
from assemblyline.al.common import forge, queue
from assemblyline.al.common import log
import logging
import os
config = forge.get_config()
config.logging.log_to_console = False
config.logging.log_to_syslog = False
config.logging.log_to_file = True
pid = str(os.getpid())
log.init_logging('reindex_worker.%s' % pid)
logger = logging.getLogger('assemblyline.reindex_worker')
# Run config
DATABASE_NUM = 3
# Globals
ds = forge.get_datastore()
reindex_queue = queue.NamedQueue('r-index', db=DATABASE_NUM)
done_queue = queue.NamedQueue("r-done", db=DATABASE_NUM)
def do_reindex(bucket_name, key):
try:
data = ds._get_bucket_item(ds.get_bucket(bucket_name), key)
data = ds.sanitize(bucket_name, data, key)
ds._save_bucket_item(ds.get_bucket(bucket_name), key, data)
#!/usr/bin/env python
import logging
import time
from assemblyline.al.common import forge
from assemblyline.al.common import log
from assemblyline.al.common.task import Task
from assemblyline.al.service.list_queue_sizes import get_service_queue_lengths
log.init_logging('plumber')
logger = logging.getLogger('assemblyline.plumber')
dispatch_queue = forge.get_dispatch_queue()
store = forge.get_datastore()
config = forge.get_config()
service_queue = {}
threshold = {}
def get_queue(n):
q = service_queue.get(n, None)
if not q:
service_queue[n] = q = forge.get_service_queue(n)
return q
for service in store.list_services():
# noinspection PyBroadException
try:
name = service.get('name')
from flask import Flask, request, session
from flask_socketio import SocketIO, emit
from assemblyline.al.common import forge, log as al_log
from assemblyline.al.common.queue import CommsQueue, NamedQueue
from assemblyline.al.common.remote_datatypes import Hash
config = forge.get_config()
datastore = forge.get_datastore()
classification = forge.get_classification()
app = Flask(__name__)
app.config['SECRET_KEY'] = config.ui.secret_key
socketio = SocketIO(app)
al_log.init_logging("ui")
AUDIT = config.ui.audit
AUDIT_LOG = logging.getLogger('assemblyline.ui.audit')
LOGGER = logging.getLogger('assemblyline.ui.socketio')
KV_SESSION = Hash("flask_sessions",
host=config.core.redis.nonpersistent.host,
port=config.core.redis.nonpersistent.port,
db=config.core.redis.nonpersistent.db)
def get_user_info(resquest_p, session_p):
uname = None
current_session = KV_SESSION.get(session_p.get("session_id", None))
if current_session:
current_session = json.loads(current_session)
#!/usr/bin/env python
import logging
import os
import signal
from assemblyline.al.common import forge
from assemblyline.al.common import log
from assemblyline.al.common import queue
config = forge.get_config()
log.init_logging('journalist')
directory = config.core.expiry.journal.directory
emptyresult_queue = queue.NamedQueue(
"ds-emptyresult",
db=config.core.redis.persistent.db,
host=config.core.redis.persistent.host,
port=config.core.redis.persistent.port,
)
logger = logging.getLogger('assemblyline.journalist')
max_open_files = 8
path_and_filehandle = []
path_to_filehandle = {}
previous = []
running = True
# noinspection PyUnusedLocal
def interrupt(unused1, unused2): # pylint:disable=W0613
#!/usr/bin/env python
import json
import logging
import signal
import sys
from assemblyline.al.common import forge
from assemblyline.al.common import log
from assemblyline.al.common import queue
config = forge.get_config()
log.init_logging('ingest-bridge')
PRODUCTION_DB = config.core.ingest_bridge.db
PRODUCTION_HOST = config.core.ingest_bridge.host
PRODUCTION_PORT = config.core.ingest_bridge.port
SEND_EVERY = int(config.core.ingest_bridge.send_every)
logger = logging.getLogger('assemblyline.ingest_bridge')
running = True
# noinspection PyUnusedLocal
def interrupt(unused1, unused2): # pylint:disable=W0613
global running # pylint:disable=W0603
logger.info("Caught signal. Coming down...")
running = False
#!/usr/bin/env python
import logging
from assemblyline.common.isotime import now_as_local
from assemblyline.al.common import forge, log as al_log
from assemblyline.al.common.heuristics import list_all_heuristics
config = forge.get_config()
al_log.init_logging('heuristic_statistics')
log = logging.getLogger('assemblyline.heuristic_statistics')
log.info("Generating heuristic statistics")
store = forge.get_datastore()
output = {"timestamp": None, "stats": None}
stats = {}
HEUR, _ = list_all_heuristics(store.list_services())
for heur in HEUR:
heur_key = heur["id"]
results = store.stats_search("result",
query='result.tags.value:"%s"' % heur["id"],
stats_fields=["result.score"])["result.score"]
if results:
heur_stat = [
results["count"], results["min"],
int(results["mean"]), results["max"]
]
