Exemplo n.º 1
0
    def send_raw(self, raw, shards=None):
        if not shards:
            config = forge.get_config()
            shards = config.core.dispatcher.shards

        task = Task(raw)
        self.send(task, shards)
Exemplo n.º 2
0
def init_logging(name='al'):
    config = forge.get_config()
    logging.root.setLevel(logging.CRITICAL)
    logger = logging.getLogger('assemblyline')
    logger.setLevel(logging.INFO)

    if config.logging.log_to_file:
        if not os.path.isdir(config.logging.directory):
            print 'Warning: log directory does not exist. Will try to create %s' % config.logging.directory
            os.makedirs(config.logging.directory)

        op_file_handler = logging.handlers.RotatingFileHandler(os.path.join(config.logging.directory, name + '.log'),
                                                               maxBytes=10485760, backupCount=5)
        op_file_handler.setLevel(logging.INFO)
        op_file_handler.setFormatter(logging.Formatter(AL_LOG_FORMAT))
        logger.addHandler(op_file_handler)

        err_file_handler = logging.handlers.RotatingFileHandler(os.path.join(config.logging.directory, name + '.err'),
                                                                maxBytes=10485760, backupCount=5)
        err_file_handler.setLevel(logging.ERROR)
        err_file_handler.setFormatter(logging.Formatter(AL_LOG_FORMAT))
        logger.addHandler(err_file_handler)
 
    if config.logging.log_to_console:
        console = logging.StreamHandler()
        console.setLevel(logging.INFO)
        console.setFormatter(logging.Formatter(AL_LOG_FORMAT))
        logger.addHandler(console)

    if config.logging.log_to_syslog and config.logging.syslog_ip:
        syslog_handler = logging.handlers.SysLogHandler(address=(config.logging.syslog_ip, 514))
        syslog_handler.formatter = logging.Formatter(AL_SYSLOG_FORMAT)
        logger.addHandler(syslog_handler)
Exemplo n.º 3
0
 def __init__(self, stub_name, log_name):
     config = forge.get_config()
     self.registry_host = config.installation.docker.get(
         'private_registry', None)
     self.log = logging.getLogger(log_name)
     self.docker_count = 0
     self.docker_contexts = {}
     self.project_id = str(uuid.uuid4()).replace("-", "")
     self.stub_name = "%s_%s_%%i" % (self.project_id, stub_name)
Exemplo n.º 4
0
def get_client(host, port, db, private):
    if not host or not port or not db:
        config = forge.get_config()
        host = host or config.core.redis.nonpersistent.host
        port = int(port or config.core.redis.nonpersistent.port)
        db = int(db or config.core.redis.nonpersistent.db)

    if private:
        return redis.StrictRedis(host=host, port=port, db=db)
    else:
        return redis.StrictRedis(connection_pool=get_pool(host, port, db))
Exemplo n.º 5
0
    def __init__(self, metrics_channel_name, logger, elastic_ip_p,
                 elastic_port_p):
        self.metrics_channel_name = metrics_channel_name
        self.elastic_ip = elastic_ip_p
        self.elastic_port = elastic_port_p
        self.scheduler = Scheduler()
        self.metrics_queue = None
        self.es = None
        self.log = logger
        self.METRIC_TYPES.update(
            forge.get_config().core.metricsd.extra_metrics)

        self.counters_lock = Lock()
        self.counters = {}
Exemplo n.º 6
0
    def validate_rules(self, rulefile, datastore=False):
        change = False
        while True:
            try:
                self.paranoid_rule_check(rulefile)
                return change
            # If something goes wrong, clean rules until valid file given
            except Exception as e:
                change = True
                if e.message.startswith('yara.SyntaxError'):

                    e_line = int(e.message.split('):', 1)[0].split("(", -1)[1])
                    e_message = e.message.split("): ", 1)[1]
                    try:
                        invalid_rule, reline = self.clean(rulefile, e_line, e_message)
                    except Exception as ve:
                        raise ve

                    # If datastore object given, change status of signature to INVALID in Riak
                    if datastore:
                        from assemblyline.al.common import forge
                        store = forge.get_datastore()
                        config = forge.get_config()
                        signature_user = config.services.master_list.Yara.config.SIGNATURE_USER
                        # Get the offending sig ID
                        sig_query = "name:{} AND meta.al_status:(DEPLOYED OR NOISY)".format(invalid_rule)
                        sigl = store.list_filtered_signature_keys(sig_query)
                        # Mark and update Riak
                        store = forge.get_datastore()
                        for sig in sigl:
                            sigdata = store.get_signature(sig)
                            # Check this in case someone already marked it as invalid
                            try:
                                if sigdata['meta']['al_status'] == 'INVALID':
                                    continue
                            except KeyError:
                                pass
                            sigdata['meta']['al_status'] = 'INVALID'
                            today = datetime.date.today().isoformat()
                            sigdata['meta']['al_state_change_date'] = today
                            sigdata['meta']['al_state_change_user'] = signature_user
                            sigdata['comments'].append("AL ERROR MSG:{0}. Line:{1}".format(e_message.rstrip().strip(),
                                                                                           reline))
                            store.save_signature(sig, sigdata)

                else:
                    raise e

                continue
Exemplo n.º 7
0
    def send(self, task, shards=None, queue_name=None):
        if queue_name is None:
            queue_name = {}

        if not shards:
            config = forge.get_config()
            shards = config.core.dispatcher.shards

        if not task.dispatch_queue:
            n = forge.determine_dispatcher(task.sid, shards)
            name = queue_name.get(n, None)
            if not name:
                queue_name[n] = name = 'ingest-queue-' + str(n)
            task.dispatch_queue = name
        if not task.priority:
            task.priority = 0
        self._get_queue(task.dispatch_queue).push(task.priority, task.raw)
Exemplo n.º 8
0
def main():
    if len(sys.argv) == 1:
        print "Usage: %s <One or more prepared VM tarballs>"
        sys.exit(7)

    try:
        svc_class = service_by_name("Cuckoo")
    except:
        print 'Could not load service "%s".\n' \
              'Valid options:\n%s' % ("Cuckoo", [s['name'] for s in forge.get_datastore().list_services()])
        sys.exit(7)

    cfg = forge.get_datastore().get_service(svc_class.SERVICE_NAME).get(
        "config", {})
    config = forge.get_config()

    local_meta_root = os.path.join(config.system.root, cfg['REMOTE_DISK_ROOT'])
    vm_meta_path = os.path.join(local_meta_root, cfg['vm_meta'])

    out_config = vm_meta_path
    out_directory = os.path.dirname(out_config)
    vm_list = sys.argv[1:]

    cuckoo_config = []
    for vm in vm_list:
        for js in install_vm_meta(out_directory, vm, ['']):
            cuckoo_config.append(js)

    with open(out_config, "w") as fh:
        json.dump(cuckoo_config,
                  fh,
                  sort_keys=True,
                  indent=4,
                  separators=(',', ': '))

    print "Wrote %i Definitions to %s!" % (len(cuckoo_config), out_config)
Exemplo n.º 9
0
import json
import logging
import platform
import re
import threading
import time

from collections import namedtuple
from assemblyline.common.importing import class_by_path
from assemblyline.al.common import forge
from assemblyline.al.common.message import Message, MT_SVCHEARTBEAT
from assemblyline.al.common.queue import CommsQueue
from assemblyline.al.service.service_driver import ServiceDriver

config = forge.get_config()
log = logging.getLogger('assemblyline.svc.mgr')

DONE = len(config.services.stages) + 1
NAME = dict([(x + 1, config.services.stages[x]) for x in xrange(len(config.services.stages))])
ORDER = dict([(config.services.stages[x], x + 1) for x in xrange(len(config.services.stages))])

ServiceEntry = namedtuple(
    'ServiceEntry', [
        'name',
        'accepts',
        'category',
        'proxy',
        'rejects',
        'skip',
        'stage',
Exemplo n.º 10
0
        Args:
            c12n: Classification to normalize
            long_format: True/False in long format
            skip_auto_select: True/False skip group auto adding, use True when dealing with user's classifications

        Returns:
            A normalized version of the original classification
        """
        if not self.enforce:
            return self.UNRESTRICTED

        lvl_idx, req, groups, subgroups = self._get_classification_parts(
            c12n, long_format=long_format)
        return self._get_normalized_classification_text(
            lvl_idx,
            req,
            groups,
            subgroups,
            long_format=long_format,
            skip_auto_select=skip_auto_select)


if __name__ == "__main__":
    import json
    from assemblyline.al.common import forge
    config = forge.get_config(
        static_seed="assemblyline.al.install.seeds.assemblyline_appliance.seed"
    )
    c = Classification(config.system.classification.definition)
    print json.dumps(c.get_parsed_classification_definition(), indent=4)
Exemplo n.º 11
0
from assemblyline.common.charset import safe_str
from assemblyline.common.isotime import epoch_to_iso
from assemblyline.al.common import forge
from assemblyline.al.common import task
from assemblyline.al.common.forge import get_constants, get_config

constants = get_constants()
config = get_config()


class NoticeException(Exception):
    pass


alert = [
    'sid',
]

aliases = config.core.alerter.metadata_aliases
meta_fields = {
    "al_score": "number",
    "filename": "text",
    "size": "number",
    "ts": "date"
}
meta_fields.update(config.core.alerter.metadata_fields)
metadata = meta_fields.keys()

overrides = task.submission_overrides + [
    'completed_queue',
    'description',
Exemplo n.º 12
0
 def submit(self, task, shards=None):
     if not shards:
         config = forge.get_config()
         shards = config.core.dispatcher.shards
     task.dispatch_queue = None
     self.send(task, shards)
Exemplo n.º 13
0
 def __init__(self, host=None, port=None, db=None):
     config = forge.get_config()
     self.host = host or config.core.redis.nonpersistent.host
     self.port = port or config.core.redis.nonpersistent.port
     self.db = db or config.core.redis.nonpersistent.db
     self.q = {}
Exemplo n.º 14
0
#!/usr/bin/env python 
import sys
import libvirt
import logging
import lxml
import lxml.etree
import os
import subprocess

from assemblyline.al.common import forge
config = forge.get_config(static_seed=os.environ.get("VMEDIT_SEED", None))
from assemblyline.al.common import vm

QCOW2_EXT = 'qcow2'
LOCAL_VMDISK_ROOT = '/opt/al/var/masterdisks/'

log = logging.getLogger('assemblyline.al.vme')


class VmEditor(object):

    def __init__(self, vmname, cfg=None):
        self.ds = forge.get_datastore()
        self.vm_name = vmname + '.001'
        if cfg:
            self.vm_cfg = cfg.workers.virtualmachines.master_list.get(vmname, {}).get('cfg', None)
        else:
            self.vm_cfg = self.ds.get_virtualmachine(vmname)
        if not self.vm_cfg:
            raise Exception("Could not find VM %s in the seed" % vmname)
        self.vmm = libvirt.open(None)
Exemplo n.º 15
0
import elasticsearch
import json
import logging
import psutil
import os
import sys
import urllib2

from apscheduler.scheduler import Scheduler

from assemblyline.common.net import get_hostname, get_hostip
from assemblyline.al.common import forge, log as al_log
from assemblyline.common.isotime import now_as_local, now_as_iso

config = forge.get_config(static_seed=os.getenv("AL_SEED_STATIC"))
al_log.init_logging('system_metrics')
log = logging.getLogger('assemblyline.system_metrics')

previous_net_io = None
previous_disk_io = None

SOLR_ADMIN_URL = '%s/admin/mbeans?stats=true&wt=json&cat=QUERYHANDLER&cat=CORE&key=/update&key=/select&key=searcher'
SOLR_CORE_URL = 'admin/cores?wt=json&indexInfo=false'
SOLR_URL_BUILDER = 'http://localhost:8093/internal_solr/%s'
RIAK_STATS_URL = 'http://localhost:8098/stats'


def calculate_system_metrics(es, cur_ip, cur_host):
    global previous_disk_io, previous_net_io
    log.info("Starting system metrics calculation...")