Beispiel #1
0
    def _dump(self, event=None):
        event_path = os.path.join(self.cur_path, 'misp_events')
        if not os.path.exists(event_path):
            os.makedirs(event_path)

        if not event:
            to_dump = __sessions__.current.misp_event.event
        elif isinstance(event, MISPEvent):
            to_dump = event
        else:
            to_dump = MISPEvent()
            to_dump.load(event)
        if to_dump.id:
            filename = str(to_dump.id)
        elif (__sessions__.is_attached_misp(True) and
                __sessions__.current.misp_event.current_dump_file):
            filename = __sessions__.current.misp_event.current_dump_file
        else:
            i = 1
            while True:
                filename = 'new_event_{}.json'.format(i)
                if not os.path.exists(os.path.join(event_path, filename)):
                    break
                i += 1

        path = os.path.join(event_path, filename)
        with open(path, 'w') as f:
            f.write(to_dump.to_json())
        self.log('success', '{} stored successfully.'.format(filename.rstrip('.json')))
        return filename
Beispiel #2
0
def create_event(self):
    if self.args.threat is not None:
        # Dirty trick to keep consistency in the module: the threat level in the upload
        # API can go from 0 import to 3 but it is 1 to 4 in the event mgmt API.
        # It will be fixed in a near future, in the meantime, we do that:
        self.args.threat += 1

    if not self.args.info:
        self.log('error', 'Info field is required for a new event')
    info = ' '.join(self.args.info)

    # Check if the following arguments have been set (and correctly set). If not, take the config values
    self.args.distrib = self.distribution if self.args.distrib is None else self.args.distrib
    self.args.sharing = self.sharinggroup if self.args.sharing is None else self.args.sharing

    if self.args.sharing and self.args.distrib != 4:
        self.args.sharing = None
        self.log('info', "Sharing group can only be set if distribution is 4. Clearing set value")

    misp_event = MISPEvent()
    misp_event.set_all_values(info=info, distribution=self.args.distrib,
                              sharing_group_id=self.args.sharing, threat_level_id=self.args.threat,
                              analysis=self.args.analysis, date=self.args.date)
    self._search_local_hashes(misp_event)
    if self.offline_mode:
        # New event created locally, no ID
        __sessions__.current.misp_event.current_dump_file = self._dump()
        __sessions__.current.misp_event.offline()
    else:
        misp_event = self.misp.add_event(json.dumps(misp_event, cls=EncodeUpdate))
        if self._has_error_message(misp_event):
            return
        __sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode))
        self._dump()
Beispiel #3
0
def load_openioc(openioc):
    # Takes a opened file, or a string
    if not has_bs4:
        raise Exception('You need to install BeautifulSoup: pip install bs4')
    misp_event = MISPEvent()
    iocreport = BeautifulSoup(openioc, "html.parser")
    # Set event fields
    info = extract_field(iocreport, 'short_description')
    if info:
        misp_event.info = info
    date = extract_field(iocreport, 'authored_date')
    if date:
        misp_event.set_date(date)
    # Set special attributes
    description = extract_field(iocreport, 'description')
    if description:
        if not misp_event.info:
            misp_event.info = description
        else:
            misp_event.add_attribute('comment', description)
    if not misp_event.info:
        misp_event.info = 'OpenIOC import'
    author = extract_field(iocreport, 'authored_by')
    if author:
        misp_event.add_attribute('comment', author)
    misp_event = set_all_attributes(iocreport, misp_event)
    return misp_event
Beispiel #4
0
    def create_event(self):
        if self.args.threat is not None:
            # Dirty trick to keep consistency in the module: the threat level in the upload
            # API can go from 0 import to 3 but it is 1 to 4 in the event mgmt API.
            # It will be fixed in a near future, in the meantime, we do that:
            self.args.threat += 1

        if not self.args.info:
            self.log('error', 'Info field is required for a new event')
        info = ' '.join(self.args.info)

        misp_event = MISPEvent()
        misp_event.set_all_values(info=info, distribution=self.args.distrib,
                                  threat_level_id=self.args.threat, analysis=self.args.analysis,
                                  date=self.args.date)
        self._search_local_hashes(misp_event)
        if self.offline_mode:
            # New event created locally, no ID
            __sessions__.current.misp_event.current_dump_file = self._dump()
            __sessions__.current.misp_event.offline()
        else:
            misp_event = self.misp.add_event(json.dumps(misp_event, cls=EncodeUpdate))
            if self._has_error_message(misp_event):
                return
            __sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode))
            self._dump()
Beispiel #5
0
 def test_eventObject(self, m):
     self.initURI(m)
     pymisp = PyMISP(self.domain, self.key)
     misp_event = MISPEvent(pymisp.describe_types)
     misp_event.load(open('tests/57c4445b-c548-4654-af0b-4be3950d210f.json', 'r').read())
     json.dumps(misp_event, cls=EncodeUpdate)
     json.dumps(misp_event, cls=EncodeFull)
Beispiel #6
0
def download(self):
    if self.offline_mode:
        self.log('error', 'Offline mode, unable to dodnload a sample')
        return
    ok = False
    data = None
    if self.args.hash:
        ok, data = self.misp.download_samples(sample_hash=self.args.hash)
    elif self.args.list is not None:
        list_events = []
        if len(self.args.list) == 0:
            event_path = os.path.join(self.cur_path, 'misp_events')
            for eid, path, title in self._get_local_events(event_path):
                list_events.append(eid)
        else:
            list_events = self.args.list

        all_data = []
        for eid in list_events:
            me = MISPEvent()
            me.load(self.misp.get(eid))
            ok, data = self.misp.download_samples(event_id=me.id)
            if not ok:
                self.log('error', data)
                continue
            if data:
                all_data += data
        data = all_data
    else:
        event_id = self._get_eventid()
        if event_id is None:
            return
        ok, data = self.misp.download_samples(event_id=event_id)

        if not ok:
            self.log('error', data)
            return
    to_print = []
    samples_path = os.path.join(self.cur_path, 'misp_samples')
    for d in data:
        eid, filename, payload = d
        path = os.path.join(samples_path, eid, filename)
        if not os.path.exists(os.path.dirname(path)):
            os.makedirs(os.path.dirname(path))
        with open(path, 'wb') as f:
            f.write(payload.getvalue())
        to_print.append((eid, path))

    if len(to_print) == 1:
        self.log('success', 'The sample has been downloaded from Event {}'.format(to_print[0][0]))
        event = self.misp.get(to_print[0][0])
        if not self._has_error_message(event):
            return __sessions__.new(to_print[0][1], MispEvent(event, self.offline_mode))
    elif len(to_print) > 1:
        self.log('success', 'The following files have been downloaded:')
        self._display_tmp_files()
    else:
        self.log('warning', 'No samples available.')
Beispiel #7
0
def create_massive_dummy_events(misp, nbattribute):
    event = MISPEvent()
    event.info = 'massive dummy event'
    event = misp.add_event(event)
    print(event)
    eventid = event.id
    distribution = '0'
    functions = [floodtxt, floodip, flooddomain, flooddomainip, floodemail, floodattachment]
    for i in range(nbattribute):
        choice = randint(0, 5)
        if choice == 5:
            floodattachment(misp, eventid, distribution, False, 'Payload delivery', '', event.info, event.analysis, event.threat_level_id)
        else:
            functions[choice](misp, event)
Beispiel #8
0
 def from_remote(self, event_id):
     from pymisp import PyMISP
     from keys import misp_url, misp_key, misp_verifycert
     misp = PyMISP(misp_url, misp_key, misp_verifycert)
     result = misp.get(event_id)
     self.misp_event = MISPEvent()
     self.misp_event.load(result)
Beispiel #9
0
def _change_event(self):
    if self.offline_mode:
        self._dump()
    else:
        if __sessions__.current.misp_event.event.id:
            event = self.misp.update(__sessions__.current.misp_event.event)
        else:
            event = self.misp.add_event(__sessions__.current.misp_event.event)
        if self._has_error_message(event):
            return
        try:
            me = MISPEvent()
            me.load(event)
            self._check_add(me)
        except Exception as e:
            self.log('error', e)
Beispiel #10
0
    def test_batch_OSINT_events(self):
        # Test case ONLY for manual testing. Needs to download a full list of OSINT events !

        if self.check_python_2():
            self.assertTrue(True)
        elif not manual_testing:
            self.assertTrue(True)
        else:
            self.init_event()

            file_nb = str(len(os.listdir(self.test_batch_folder)))
            i = 0
            t = time.time()
            for curr_file in os.listdir(self.test_batch_folder):
                self.mispevent = MISPEvent()
                file_path = self.test_batch_folder + curr_file

                print("Current file : " + file_path + " " + str(i) + " over " + file_nb)
                i += 1

                self.mispevent.load_file(file_path)

                reportlab_generator.register_value_to_file(
                    reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
                    self.storage_folder_OSINT + curr_file + ".pdf")
            print("Elapsed time : " + str(time.time() - t))
Beispiel #11
0
    def test_batch_OSINT_with_config_events(self):
        # Test case ONLY for manual testing. Needs to download a full list of OSINT events !

        if self.check_python_2():
            self.assertTrue(True)
        elif not manual_testing:
            self.assertTrue(True)
        else:
            self.init_event()

            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"
            config[self.moduleconfig[2]] = True
            config[self.moduleconfig[3]] = True
            config[self.moduleconfig[4]] = True
            config[self.moduleconfig[5]] = True

            file_nb = str(len(os.listdir(self.test_batch_folder)))
            i = 0
            t = time.time()
            for curr_file in os.listdir(self.test_batch_folder):
                self.mispevent = MISPEvent()
                file_path = self.test_batch_folder + curr_file

                print("Current file : " + file_path + " " + str(i) + " over " + file_nb)
                i += 1

                self.mispevent.load_file(file_path)

                reportlab_generator.register_value_to_file(
                    reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                    self.storage_folder_OSINT + curr_file + ".pdf")
            print("Elapsed time : " + str(time.time() - t))
Beispiel #12
0
class MispEvent(object):
    def __init__(self, event, offline=False):
        if isinstance(event, MISPEvent):
            self.event = event
        else:
            self.event = MISPEvent()
            if isinstance(event, six.string_types) and os.path.exists(event):
                self.event.load_file(event)
            else:
                self.event.load(event)
        self.off = offline
        if self.event.id:
            self.current_dump_file = '{}.json'.format(self.event.id)
        else:
            self.current_dump_file = None

    def online(self):
        self.off = False

    def offline(self):
        self.off = True

    def get_all_ips(self):
        return [a.value for a in self.event.attributes if a.type in ['ip-dst', 'ip-src']]

    def get_all_domains(self):
        return [a.value for a in self.event.attributes if a.type in ['domain', 'hostname']]

    def get_all_urls(self):
        return [a.value for a in self.event.attributes if a.type == 'url']

    def get_all_hashes(self):
        event_hashes = []
        sample_hashes = []
        for a in self.event.attributes:
            h = None
            if a.type in ('md5', 'sha1', 'sha256'):
                h = a.value
                event_hashes.append(h)
            elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'):
                h = a.value.split('|')[1]
                event_hashes.append(h)
            elif a.type == 'malware-sample':
                h = a.value.split('|')[1]
                sample_hashes.append(h)
        return event_hashes, sample_hashes
Beispiel #13
0
 def __init__(self, event, offline=False):
     if isinstance(event, MISPEvent):
         self.event = event
     else:
         self.event = MISPEvent()
         self.event.load(event)
     self.off = offline
     if self.event.id:
         self.current_dump_file = '{}.json'.format(self.event.id)
     else:
         self.current_dump_file = None
Beispiel #14
0
    def _search(self, query):
        if self.offline_mode:
            self.log('error', 'Offline mode, unable to search')
            return
        result = self.misp.search_all(query)

        if self._has_error_message(result):
            return
        self.log('success', '{} matches on the following events:'.format(query))
        for e in result['response']:
            nb_samples = 0
            nb_hashes = 0
            me = MISPEvent()
            me.load(e)
            for a in me.attributes + [attribute for obj in me.objects for attribute in obj.attributes]:
                if a.type == 'malware-sample':
                    nb_samples += 1
                if a.type in ('md5', 'sha1', 'sha256', 'filename|md5', 'filename|sha1', 'filename|sha256'):
                    nb_hashes += 1
            self.log('item', '{} ({} samples, {} hashes) - {}{}{}'.format(me.info, nb_samples, nb_hashes, self.url, '/events/view/', me.id))
Beispiel #15
0
def create_new_event():
    me = MISPEvent()
    me.info = "Fail2Ban blocking"
    me.add_tag(args.tag)
    start = datetime.now()
    me.add_attribute('datetime', start.isoformat(), comment='Start Time')
    return me
Beispiel #16
0
 def _search_local_hashes(self, event, open_session=True):
     local = []
     samples_count = 0
     if isinstance(event, MISPEvent):
         misp_event = event
     elif event.get('Event') is None:
         self.log('error', event)
         return
     else:
         misp_event = MISPEvent()
         misp_event.load(event)
     if not hasattr(misp_event, 'id'):
         # The event doesn't exists upstream, breaking.
         return
     for a in misp_event.attributes + [attribute for obj in misp_event.objects for attribute in obj.attributes]:
         row = None
         if a.type == 'malware-sample':
             samples_count += 1
         if a.type in ('md5', 'sha1', 'sha256'):
             row = Database().find(key=a.type, value=a.value)
         elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'):
             row = Database().find(key=a.type.split('|')[1], value=a.value.split('|')[1])
         elif a.type == 'malware-sample':
             row = Database().find(key='md5', value=a.value.split('|')[1])
         if row:
             local.append(row[0])
     self.log('info', 'Event {} contains {} samples.'.format(misp_event.id, samples_count))
     if not open_session:
         return
     shas = set([l.sha256 for l in local])
     if len(shas) == 1:
         __sessions__.new(get_sample_path(shas.pop()), MispEvent(misp_event, self.offline_mode))
     elif len(shas) > 1:
         self.log('success', 'The following samples are in this viper instance:')
         __sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode))
         for s in shas:
             self.log('item', s)
     else:
         __sessions__.new(misp_event=MispEvent(misp_event, self.offline_mode))
         self.log('info', 'No known (in Viper) samples in that event.')
Beispiel #17
0
 def __init__(self, event, offline=False):
     if isinstance(event, MISPEvent):
         self.event = event
     else:
         self.event = MISPEvent()
         if isinstance(event, six.string_types) and os.path.exists(event):
             self.event.load_file(event)
         else:
             self.event.load(event)
     self.off = offline
     if self.event.id:
         self.current_dump_file = '{}.json'.format(self.event.id)
     else:
         self.current_dump_file = None
Beispiel #18
0
 def setUp(self):
     self.maxDiff = None
     self.mispevent = MISPEvent()
     if not manual_testing:
         self.root = "tests/"
     else:
         self.root = ""
     self.test_folder = self.root + "reportlab_testfiles/"
     self.test_batch_folder = self.root + "OSINT_output/"
     self.storage_folder_OSINT = self.root + "OSINT_PDF/"
     self.test_image_folder = self.root + "image_json/"
     self.storage_folder = self.root + "reportlab_testoutputs/"
     self.storage_image_folder = self.root + "reportlab_test_image_outputs/"
     self.moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description",
                          "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"]
Beispiel #19
0
def load_openioc(openioc):
    if not has_bs4:
        raise Exception('You need to install BeautifulSoup: pip install bs4')
    misp_event = MISPEvent()
    with open(openioc, "r") as ioc_file:
        iocreport = BeautifulSoup(ioc_file, "lxml")
        # Set event fields
        info = extract_field(iocreport, 'short_description')
        if info:
            misp_event.info = info
        date = extract_field(iocreport, 'authored_date')
        if date:
            misp_event.set_date(date)
        # Set special attributes
        description = extract_field(iocreport, 'description')
        if description:
            misp_event.add_attribute('comment', description)
        author = extract_field(iocreport, 'authored_by')
        if author:
            misp_event.add_attribute('comment', author)
        misp_event = set_all_attributes(iocreport, misp_event)
    return misp_event
Beispiel #20
0
    def store(self):
        try:
            event_path = os.path.join(self.cur_path, 'misp_events')
            if not os.path.exists(event_path):
                os.mkdir(event_path)
            if self.args.list:
                header = ['Event ID', 'Title']
                rows = []
                for eid, path, title in self._get_local_events(event_path):
                    rows.append((eid, title))
                self.log('table', dict(header=header, rows=sorted(rows, key=lambda i: (int(i[0].split('_')[-1])))))
            elif self.args.update:
                if self.offline_mode:
                    self.log('error', 'Offline mode, cannot update locally stored events.')
                    return
                for eid, path, title in self._get_local_events(event_path):
                    event = self.misp.get(eid)
                    with open(path, 'w') as f:
                        f.write(json.dumps(event))
                    self.log('success', '{} updated successfully.'.format(eid))
            elif self.args.sync:
                if self.offline_mode:
                    self.log('error', 'Offline mode, cannot synchronize locally stored events.')
                    return
                for eid, path, title in self._get_local_events(event_path):
                    __sessions__.close()
                    event = MISPEvent()
                    event.load(path)
                    if 'new_event_' in path:
                        event = self.misp.add_event(json.dumps(event, cls=EncodeUpdate))
                        try:
                            self._dump(event)
                            os.remove(path)
                        except Exception as e:
                            self.log('error', 'Unable to create new event: {}.'.format(e))
                    else:
                        eid = event.id
                        try:
                            event = self.misp.update(event._json())
                        except Exception as e:
                            self.log('error', 'Unable to update event {}: {}.'.format(eid, e))

                    if self._has_error_message(event):
                        return
            elif self.args.delete:
                path = os.path.join(event_path, '{}.json'.format(self.args.delete))
                if os.path.exists(path):
                    os.remove(path)
                    self.log('success', '{} removed successfully.'.format(self.args.delete))
                else:
                    self.log('error', '{} does not exists.'.format(self.args.delete))
            elif self.args.open:
                filename = '{}.json'.format(self.args.open)
                path = os.path.join(event_path, filename)
                if os.path.exists(path):
                    try:
                        with open(path, 'r') as f:
                            e_json = json.load(f)
                        __sessions__.new(misp_event=MispEvent(e_json, self.offline_mode))
                        __sessions__.current.misp_event.current_dump_file = filename
                    except Exception as e:
                        self.log('error', 'Unable to open {}: {}'.format(path, e))
                else:
                    self.log('error', '{} does not exists.'.format(self.args.open))
            elif __sessions__.is_attached_misp():
                self._dump()
        except IOError as e:
            self.log('error', e.strerror)
Beispiel #21
0
#!/usr/bin/env python
# -*- coding: utf-8 -*-

from pymisp import ExpandedPyMISP, MISPEvent
from keys import misp_url, misp_key, misp_verifycert
import argparse

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Update a MISP event.")
    parser.add_argument("-e",
                        "--event",
                        required=True,
                        help="Event ID to update.")
    parser.add_argument("-i", "--input", required=True, help="Input file")

    args = parser.parse_args()

    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)

    me = MISPEvent()
    me.load_file(args.input)

    result = misp.update_event(args.event, me)
Beispiel #22
0
class Yeti():
    def __init__(self, url, key, attribute):
        self.misp_mapping = {
            'Ip': 'ip-dst',
            'Domain': 'domain',
            'Hostname': 'hostname',
            'Url': 'url',
            'AutonomousSystem': 'AS',
            'File': 'sha256'
        }
        self.yeti_client = pyeti.YetiApi(url=url, api_key=key)
        self.attribute = attribute
        self.misp_event = MISPEvent()
        self.misp_event.add_attribute(**attribute)

    def search(self, value):
        obs = self.yeti_client.observable_search(value=value)
        if obs:
            return obs[0]

    def get_neighboors(self, obs_id):
        neighboors = self.yeti_client.neighbors_observables(obs_id)
        if neighboors and 'objs' in neighboors:
            links_by_id = {
                link['dst']['id']: (link['description'], 'dst')
                for link in neighboors['links'] if link['dst']['id'] != obs_id
            }
            links_by_id.update({
                link['src']['id']: (link['description'], 'src')
                for link in neighboors['links'] if link['src']['id'] != obs_id
            })

            for n in neighboors['objs']:
                yield n, links_by_id[n['id']]

    def parse_yeti_result(self):
        obs = self.search(self.attribute['value'])

        for obs_to_add, link in self.get_neighboors(obs['id']):
            object_misp_domain_ip = self.__get_object_domain_ip(obs_to_add)
            if object_misp_domain_ip:
                self.misp_event.add_object(object_misp_domain_ip)
                continue
            object_misp_url = self.__get_object_url(obs_to_add)
            if object_misp_url:
                self.misp_event.add_object(object_misp_url)
                continue
            if link[0] == 'NS record':
                object_ns_record = self.__get_object_ns_record(
                    obs_to_add, link[1])
                if object_ns_record:
                    self.misp_event.add_object(object_ns_record)
                    continue
            self.__get_attribute(obs_to_add, link[0])

    def get_result(self):
        event = json.loads(self.misp_event.to_json())
        results = {
            key: event[key]
            for key in ('Attribute', 'Object') if key in event
        }
        return results

    def __get_attribute(self, obs_to_add, link):

        try:
            type_attr = self.misp_mapping[obs_to_add['type']]
            value = None
            if obs_to_add['type'] == 'File':
                value = obs_to_add['value'].split(':')[1]
            else:
                value = obs_to_add['value']
            attr = self.misp_event.add_attribute(value=value, type=type_attr)
            attr.comment = '%s: %s' % (link, self.attribute['value'])
        except KeyError:
            logging.error('type not found %s' % obs_to_add['type'])
            return

        for t in obs_to_add['tags']:
            self.misp_event.add_attribute_tag(t['name'], attr['uuid'])

    def __get_object_domain_ip(self, obj_to_add):
        if (obj_to_add['type'] == 'Ip' and self.attribute['type'] in ['hostname', 'domain']) or \
                (obj_to_add['type'] in ('Hostname', 'Domain') and self.attribute['type'] in ('ip-src', 'ip-dst')):
            domain_ip_object = MISPObject('domain-ip')
            domain_ip_object.add_attribute(self.__get_relation(obj_to_add),
                                           obj_to_add['value'])
            domain_ip_object.add_attribute(
                self.__get_relation(self.attribute, is_yeti_object=False),
                self.attribute['value'])
            domain_ip_object.add_reference(self.attribute['uuid'],
                                           'related_to')

            return domain_ip_object

    def __get_object_url(self, obj_to_add):
        if (obj_to_add['type'] == 'Url' and self.attribute['type'] in [
                'hostname', 'domain', 'ip-src', 'ip-dst'
        ]) or (obj_to_add['type'] in ('Hostname', 'Domain', 'Ip')
               and self.attribute['type'] == 'url'):
            url_object = MISPObject('url')
            obj_relation = self.__get_relation(obj_to_add)
            if obj_relation:
                url_object.add_attribute(obj_relation, obj_to_add['value'])
            obj_relation = self.__get_relation(self.attribute,
                                               is_yeti_object=False)
            if obj_relation:
                url_object.add_attribute(obj_relation, self.attribute['value'])
            url_object.add_reference(self.attribute['uuid'], 'related_to')

            return url_object

    def __get_object_ns_record(self, obj_to_add, link):
        queried_domain = None
        ns_domain = None
        object_dns_record = MISPObject('dns-record')
        if link == 'dst':
            queried_domain = self.attribute['value']
            ns_domain = obj_to_add['value']
        elif link == 'src':
            queried_domain = obj_to_add['value']
            ns_domain = self.attribute['value']
        if queried_domain and ns_domain:
            object_dns_record.add_attribute('queried-domain', queried_domain)
            object_dns_record.add_attribute('ns-record', ns_domain)
            object_dns_record.add_reference(self.attribute['uuid'],
                                            'related_to')

            return object_dns_record

    def __get_relation(self, obj, is_yeti_object=True):
        if is_yeti_object:
            type_attribute = self.misp_mapping[obj['type']]
        else:
            type_attribute = obj['type']
        if type_attribute == 'ip-src' or type_attribute == 'ip-dst':
            return 'ip'
        elif 'domain' == type_attribute:
            return 'domain'
        elif 'hostname' == type_attribute:
            return 'domain'
        elif type_attribute == 'url':
            return type_attribute
Beispiel #23
0
class RFEnricher:
    """Class for enriching an attribute with data from Recorded Future.
    The enrichment data is returned as a custom MISP object.
    """
    def __init__(self, attribute_props: dict):
        self.event = MISPEvent()
        self.enrichment_object = MISPObject("Recorded Future Enrichment")
        description = ("An object containing the enriched attribute and "
                       "related entities from Recorded Future.")
        self.enrichment_object.from_dict(**{
            "meta-category": "misc",
            "description": description,
            "distribution": 0
        })

        # Create a copy of enriched attribute to add tags to
        temp_attr = MISPAttribute()
        temp_attr.from_dict(**attribute_props)
        self.enriched_attribute = MISPAttribute()
        self.enriched_attribute.from_dict(**{
            "value": temp_attr.value,
            "type": temp_attr.type,
            "distribution": 0
        })

        self.related_attributes: List[Tuple[str, MISPAttribute]] = []
        self.color_picker = RFColors()
        self.galaxy_finder = GalaxyFinder()

        # Mapping from MISP-type to RF-type
        self.type_to_rf_category = {
            "ip": "ip",
            "ip-src": "ip",
            "ip-dst": "ip",
            "ip-src|port": "ip",
            "ip-dst|port": "ip",
            "domain": "domain",
            "hostname": "domain",
            "md5": "hash",
            "sha1": "hash",
            "sha256": "hash",
            "uri": "url",
            "url": "url",
            "vulnerability": "vulnerability",
            "weakness": "vulnerability",
        }

        # Related entities have 'Related' as part of the word and Links entities from RF
        # portrayed as related attributes in MISP
        self.related_attribute_types = [
            "RelatedIpAddress",
            "RelatedInternetDomainName",
            "RelatedHash",
            "RelatedEmailAddress",
            "RelatedCyberVulnerability",
            "IpAddress",
            "InternetDomainName",
            "Hash",
            "EmailAddress",
            "CyberVulnerability",
        ]
        # Related entities have 'Related' as part of the word and and Links entities from RF portrayed as tags in MISP
        self.galaxy_tag_types = [
            "RelatedMalware",
            "RelatedThreatActor",
            "Threat Actor",
            "MitreAttackIdentifier",
            "Malware",
        ]

    def enrich(self) -> None:
        """Run the enrichment."""
        category = self.type_to_rf_category.get(self.enriched_attribute.type,
                                                "")
        enriched_attribute_value = self.enriched_attribute.value
        # If enriched attribute has a port we need to remove that port
        # since RF do not support enriching ip addresses with port
        if self.enriched_attribute.type in ["ip-src|port", "ip-dst|port"]:
            enriched_attribute_value = enriched_attribute_value.split("|")[0]
        json_response = GLOBAL_REQUEST_HANDLER.rf_lookup(
            category, enriched_attribute_value)
        response = json.loads(json_response.content)

        try:
            # Add risk score and risk rules as tags to the enriched attribute
            risk_score = response["data"]["risk"]["score"]
            hex_color = self.color_picker.riskscore_color(risk_score)
            tag_name = f'recorded-future:risk-score="{risk_score}"'
            self.add_tag(tag_name, hex_color)
            risk_criticality = response["data"]["risk"]["criticalityLabel"]
            hex_color = self.color_picker.criticality_color(risk_criticality)
            tag_name = f'recorded-future:criticality="{risk_criticality}"'
            self.add_tag(tag_name, hex_color)

            for evidence in response["data"]["risk"]["evidenceDetails"]:
                risk_rule = evidence["rule"]
                criticality = evidence["criticality"]
                hex_color = self.color_picker.riskrule_color(criticality)
                tag_name = f'recorded-future:risk-rule="{risk_rule}"'
                self.add_tag(tag_name, hex_color)

            links_data = response["data"].get("links", {}).get("hits")
            # Check if we have error in links response. If yes, then user do not have right module enabled in token
            links_access_error = response["data"].get("links", {}).get("error")
            galaxy_tags = []
            if not links_access_error:
                for hit in links_data:
                    for section in hit["sections"]:
                        for sec_list in section["lists"]:
                            entity_type = sec_list["type"]["name"]
                            for entity in sec_list["entities"]:
                                if entity_type in self.galaxy_tag_types:
                                    galaxy = self.galaxy_finder.find_galaxy_match(
                                        entity["name"], entity_type)
                                    if galaxy and galaxy not in galaxy_tags:
                                        galaxy_tags.append(galaxy)
                                else:
                                    self.add_attribute(entity["name"],
                                                       entity_type)

            else:
                # Retrieve related entities
                for related_entity in response["data"]["relatedEntities"]:
                    related_type = related_entity["type"]
                    if related_type in self.related_attribute_types:
                        # Related entities returned as additional attributes
                        for related in related_entity["entities"]:
                            # filter those entities that have count bigger than 4, to reduce noise
                            # because there can be a huge list of related entities
                            if int(related["count"]) > 4:
                                indicator = related["entity"]["name"]
                                self.add_attribute(indicator, related_type)
                    elif related_type in self.galaxy_tag_types:
                        # Related entities added as galaxy-tags to the enriched attribute
                        galaxy_tags = []
                        for related in related_entity["entities"]:
                            # filter those entities that have count bigger than 4, to reduce noise
                            # because there can be a huge list of related entities
                            if int(related["count"]) > 4:
                                indicator = related["entity"]["name"]
                                galaxy = self.galaxy_finder.find_galaxy_match(
                                    indicator, related_type)
                                # Handle deduplication of galaxy tags
                                if galaxy and galaxy not in galaxy_tags:
                                    galaxy_tags.append(galaxy)
            for galaxy in galaxy_tags:
                self.add_tag(galaxy)

        except KeyError:
            misperrors[
                "error"] = "Unexpected format in Recorded Future api response."
            raise

    def add_attribute(self, indicator: str, indicator_type: str) -> None:
        """Helper method for adding an indicator to the attribute list."""
        out_type = self.get_output_type(indicator_type, indicator)
        attribute = MISPAttribute()
        attribute.from_dict(**{
            "value": indicator,
            "type": out_type,
            "distribution": 0
        })
        self.related_attributes.append((indicator_type, attribute))

    def add_tag(self, tag_name: str, hex_color: str = None) -> None:
        """Helper method for adding a tag to the enriched attribute."""
        tag = MISPTag()
        tag_properties = {"name": tag_name}
        if hex_color:
            tag_properties["colour"] = hex_color
        tag.from_dict(**tag_properties)
        self.enriched_attribute.add_tag(tag)

    def get_output_type(self, related_type: str, indicator: str) -> str:
        """Helper method for translating a Recorded Future related type to a MISP output type."""
        output_type = "text"
        if related_type in ["RelatedIpAddress", "IpAddress"]:
            output_type = "ip-dst"
        elif related_type in [
                "RelatedInternetDomainName", "InternetDomainName"
        ]:
            output_type = "domain"
        elif related_type in ["RelatedHash", "Hash"]:
            hash_len = len(indicator)
            if hash_len == 64:
                output_type = "sha256"
            elif hash_len == 40:
                output_type = "sha1"
            elif hash_len == 32:
                output_type = "md5"
        elif related_type in ["RelatedEmailAddress", "EmailAddress"]:
            output_type = "email-src"
        elif related_type in [
                "RelatedCyberVulnerability", "CyberVulnerability"
        ]:
            signature = indicator.split("-")[0]
            if signature == "CVE":
                output_type = "vulnerability"
            elif signature == "CWE":
                output_type = "weakness"
        elif related_type == "MalwareSignature":
            output_type = "malware-sample"
        elif related_type == "Organization":
            output_type = "target-org"
        elif related_type == "Username":
            output_type = "target-user"
        return output_type

    def get_results(self) -> dict:
        """Build and return the enrichment results."""
        self.enrichment_object.add_attribute("Enriched attribute",
                                             **self.enriched_attribute)
        for related_type, attribute in self.related_attributes:
            self.enrichment_object.add_attribute(related_type, **attribute)
        self.event.add_object(**self.enrichment_object)
        event = json.loads(self.event.to_json())
        result = {key: event[key] for key in ["Object"] if key in event}
        return {"results": result}
Beispiel #24
0
 def handle_url(self, url, event: pymisp.MISPEvent):
     """Handle a single URL."""
     event.add_attribute("url", str(url))
     return event
Beispiel #25
0
 def handle_domain(self, domain, event: pymisp.MISPEvent):
     """Handle a single domain."""
     event.add_attribute("domain", str(domain))
     return event
Beispiel #26
0
    def import_all(self, stations_short_names, interval, data_type):
        object_creator = getattr(self, f'{interval}_flask_{data_type}')
        if data_type == 'co2':
            base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_co2/'
        elif data_type in ['c13', 'o18']:
            base_url = 'https://scrippsco2.ucsd.edu/assets/data/atmospheric/stations/flask_isotopic/'
        for station in stations_short_names:
            url = f'{base_url}/{interval}/{interval}_flask_{data_type}_{station}.csv'
            infofield = f'[{station.upper()}] {interval} average atmospheric {data_type} concentrations'
            filepath = self.fetch(url)
            if not filepath:
                continue
            if infofield in self.scrippts_meta:
                event = MISPEvent()
                event.load_file(str(self.output_dir / self.scrippts_meta[infofield]))
                location = event.get_objects_by_name('geolocation')[0]
                update = True
            else:
                event = MISPEvent()
                event.uuid = str(uuid4())
                event.info = infofield
                event.Orgc = self.misp_org
                event.add_tag(getattr(self, f'tag_{station}')())
                location = getattr(self, f'geolocation_{station}')()
                event.add_object(location)
                event.add_attribute('link', f'https://scrippsco2.ucsd.edu/data/atmospheric_co2/{station}')
                update = False
                with self.scrippts_meta_file.open('a') as f:
                    writer = csv.writer(f)
                    writer.writerow([infofield, f'{event.uuid}.json'])

            object_creator(event, location, filepath, update)
            if update:
                # Bump the publish timestamp
                event.publish_timestamp = datetime.datetime.timestamp(datetime.datetime.now())
            feed_output = event.to_feed(with_meta=False)
            with (self.output_dir / f'{event.uuid}.json').open('w') as f:
                # json.dump(feed_output, f, indent=2, sort_keys=True)  # For testing
                json.dump(feed_output, f)
Beispiel #27
0
 def from_file(self, path):
     self.misp_event = MISPEvent()
     self.misp_event.load_file(path)
Beispiel #28
0
    def run(self, results):
        """Run analysis.
        @return: MISP results dict.
        """

        url = self.options.get("url", "")
        apikey = self.options.get("apikey", "")

        if not url or not apikey:
            log.error("MISP URL or API key not configured.")
            return

        self.misp = PyMISP(url, apikey, False, "json")

        self.threads = self.options.get("threads", "")
        if not self.threads:
            self.threads = 5

        self.iocs = deque()
        self.misper = dict()

        try:
            if self.options.get("upload_iocs", False) and results.get("malscore", 0) >= self.options.get("min_malscore", 0):
                distribution = int(self.options.get("distribution", 0))
                threat_level_id = int(self.options.get("threat_level_id", 4))
                analysis = int(self.options.get("analysis", 0))
                tag = self.options.get("tag") or "CAPEv2"
                info = self.options.get("title", "")
                upload_sample = self.options.get("upload_sample")

                malfamily = ""
                if results.get("detections", ""):
                    malfamily = results["detections"]

                response = self.misp.search("attributes", value=results["target"]["file"]["sha256"], return_format="json", pythonify=True)
                if response:
                    event = self.misp.get_event(response[0].event_id, pythonify=True)
                else:
                    event = MISPEvent()
                    event.distribution = distribution
                    event.threat_level_id = threat_level_id
                    event.analysis = analysis
                    event.info = "{} {} - {}".format(info, malfamily, results.get("info", {}).get("id"))
                    event = self.misp.add_event(event, pythonify=True)

                # Add a specific tag to flag Cuckoo's event
                if tag:
                    self.misp.tag(event, tag)

                # malpedia galaxy
                if malpedia_json:
                    self.malpedia(results, event, malfamily)

                # ToDo?
                self.signature(results, event)

                self.sample_hashes(results, event)
                self.all_network(results, event)
                self.dropped_files(results, event)

                if upload_sample:
                    target = results.get("target", {})
                    f = target.get("file", {})
                    if target.get("category") == "file" and f:
                        with open(f["path"], "rb") as f:
                            event.add_attribute(
                                "malware-sample",
                                value=os.path.basename(f["path"]),
                                data=BytesIO(f.read()),
                                expand="binary",
                                comment="Sample run",
                            )

                if results.get("target", {}).get("url", "") and results["target"]["url"] not in whitelist:
                    event.add_attribute("url", results["target"]["url"])

                # ToDo migth be outdated!
                # if self.options.get("ids_files", False) and "suricata" in results.keys():
                #    for surifile in results["suricata"]["files"]:
                #        if "file_info" in surifile.keys():
                #            self.misper["iocs"].append({"md5": surifile["file_info"]["md5"]})
                #            self.misper["iocs"].append({"sha1": surifile["file_info"]["sha1"]})
                #            self.misper["iocs"].append({"sha256": surifile["file_info"]["sha256"]})

                if self.options.get("mutexes", False) and "behavior" in results and "summary" in results["behavior"]:
                    if "mutexes" in results.get("behavior", {}).get("summary", {}):
                        for mutex in results["behavior"]["summary"]["mutexes"]:
                            if mutex not in whitelist:
                                event.add_attribute("mutex", mutex)

                if self.options.get("registry", False) and "behavior" in results and "summary" in results["behavior"]:
                    if "read_keys" in results["behavior"].get("summary", {}):
                        for regkey in results["behavior"]["summary"]["read_keys"]:
                            event.add_attribute("regkey", regkey)

                event.run_expansions()
                self.misp.update_event(event)

                # Make event public
                if self.options.get("published", True):
                    self.misp.publish(event)

        except Exception as e:
            log.error("Failed to generate JSON report: %s" % e, exc_info=True)
Beispiel #29
0
def create_misp_event(misp_instance, isight_report_instance, event_tags):
    # No MISP event for this iSight report ID exists yet.
    # Alas, create a new MISP event.

    # Convert the publication date of the iSight report into a datetime object.
    if isight_report_instance.publishDate:
        date = datetime.datetime.fromtimestamp(
            isight_report_instance.publishDate)
    else:
        # If iSight doesn't provide a date, use today's date.
        date = datetime.datetime.now(datetime.timezone.utc)

    # Create a MISP event from the FireEye iSight report with the following parameters.
    event = MISPEvent()
    event.distribution = 1  # This community only
    if isight_report_instance.riskRating == 'CRITICAL' or isight_report_instance.riskRating == 'Critical':
        event.threat_level_id = 1  # High
    elif isight_report_instance.riskRating == 'HIGH' or isight_report_instance.riskRating == 'High':
        event.threat_level_id = 1  # High
    elif isight_report_instance.riskRating == 'MEDIUM' or isight_report_instance.riskRating == 'Medium':
        event.threat_level_id = 2  # Medium
    elif isight_report_instance.riskRating == 'LOW' or isight_report_instance.riskRating == 'Low':
        event.threat_level_id = 3  # Low
    else:
        event.threat_level_id = 4  # Unknown
    event.analysis = 2  # Completed
    event.info = "iSIGHT: " + isight_report_instance.title
    event.date = date

    # Push the event to the MISP server.
    my_event = misp_instance.add_event(event, pythonify=True)
    PySight_settings.logger.debug('Created MISP event %s for iSight report %s',
                                  event, isight_report_instance.reportId)
    # Add the event ID to the global list of newly created events.
    global new_events
    new_events.append(my_event['id'])

    # Add default tags to the event.
    if event_tags:
        for event_tag in event_tags:
            misp_instance.tag(my_event, event_tag)

    # Use some iSight ThreatScapes for event tagging. Reports can have multiple ThreatScapes.
    if 'Cyber Espionage' in isight_report_instance.ThreatScape:
        # VERIS distinguishes between external, internal or partner actors. This difference is not yet implemented in
        # MISP. External would be most likely.
        #misp_instance.tag(my_event, 'veris:actor:external:motive="Espionage"')
        misp_instance.tag(my_event, 'veris:actor:motive="Espionage"')
    if 'Hacktivism' in isight_report_instance.ThreatScape:
        misp_instance.tag(my_event, 'veris:actor:external:variety="Activist"')
    if 'Critical Infrastructure' in isight_report_instance.ThreatScape:
        misp_instance.tag(my_event, 'basf:technology="OT"')
    if 'Cyber Physical' in isight_report_instance.ThreatScape:
        misp_instance.tag(my_event, 'basf:technology="OT"')
    if 'Cyber Crime' in isight_report_instance.ThreatScape:
        misp_instance.tag(my_event,
                          'veris:actor:external:variety="Organized crime"')

    # Add the iSight report ID and web link as attributes.
    if isight_report_instance.reportId:
        misp_instance.add_attribute(my_event, {
            'category': 'External analysis',
            'type': 'text',
            'to_ids': False,
            'value': isight_report_instance.reportId
        },
                                    pythonify=True)
    if isight_report_instance.webLink:
        misp_instance.add_attribute(my_event, {
            'category': 'External analysis',
            'type': 'link',
            'to_ids': False,
            'value': isight_report_instance.webLink
        },
                                    pythonify=True)

    # Put the ThreatScape into an Attribution attribute, but disable correlation.
    if isight_report_instance.ThreatScape:
        misp_instance.add_attribute(my_event, {
            'category': 'Attribution',
            'type': 'text',
            'to_ids': False,
            'value': isight_report_instance.ThreatScape,
            'disable_correlation': True
        },
                                    pythonify=True)

    # Add specific attributes from this iSight report.
    update_misp_event(misp_instance, my_event, isight_report_instance)
Beispiel #30
0
def parse_response(censys_output, attribute):
    misp_event = MISPEvent()
    misp_event.add_attribute(**attribute)
    # Generic fields (for IP/Websites)
    if "autonomous_system" in censys_output:
        cen_as = censys_output['autonomous_system']
        asn_object = MISPObject('asn')
        asn_object.add_attribute('asn', value=cen_as["asn"])
        asn_object.add_attribute('description', value=cen_as['name'])
        asn_object.add_attribute('subnet-announced',
                                 value=cen_as['routed_prefix'])
        asn_object.add_attribute('country', value=cen_as['country_code'])
        asn_object.add_reference(attribute.uuid, 'associated-to')
        misp_event.add_object(**asn_object)

    if "ip" in censys_output and "ports" in censys_output:
        ip_object = MISPObject('ip-port')
        ip_object.add_attribute('ip', value=censys_output['ip'])
        for p in censys_output['ports']:
            ip_object.add_attribute('dst-port', value=p)
        ip_object.add_reference(attribute.uuid, 'associated-to')
        misp_event.add_object(**ip_object)

    # We explore all ports to find https or ssh services
    for k in censys_output.keys():
        if not isinstance(censys_output[k], dict):
            continue
        if 'https' in censys_output[k]:
            try:
                cert = censys_output[k]['https']['tls']['certificate']
                cert_obj = get_certificate_object(cert, attribute)
                misp_event.add_object(**cert_obj)
            except KeyError:
                print("Error !")
        if 'ssh' in censys_output[k]:
            try:
                cert = censys_output[k]['ssh']['v2']['server_host_key']
                # TODO enable once the type is merged
                # misp_event.add_attribute(type='hasshserver-sha256', value=cert['fingerprint_sha256'])
            except KeyError:
                pass

    # Info from certificate query
    if "parsed" in censys_output:
        cert_obj = get_certificate_object(censys_output, attribute)
        misp_event.add_object(**cert_obj)

    # Location can be present for IP/Websites results
    if "location" in censys_output:
        loc_obj = MISPObject('geolocation')
        loc = censys_output['location']
        loc_obj.add_attribute('latitude', value=loc['latitude'])
        loc_obj.add_attribute('longitude', value=loc['longitude'])
        if 'city' in loc:
            loc_obj.add_attribute('city', value=loc['city'])
        loc_obj.add_attribute('country', value=loc['country'])
        if 'postal_code' in loc:
            loc_obj.add_attribute('zipcode', value=loc['postal_code'])
        if 'province' in loc:
            loc_obj.add_attribute('region', value=loc['province'])
        loc_obj.add_reference(attribute.uuid, 'associated-to')
        misp_event.add_object(**loc_obj)

    event = json.loads(misp_event.to_json())
    return {'Object': event['Object'], 'Attribute': event['Attribute']}
Beispiel #31
0
    def __init__(self):
        super(MISP, self).__init__()
        self.cur_path = __project__.get_path()
        self.parser.add_argument("--url", help='URL of the MISP instance')
        self.parser.add_argument("--off", action='store_true', help='Use offline (can only work on pre-downloaded events)')
        self.parser.add_argument("--on", action='store_true', help='Switch to online mode')
        self.parser.add_argument("-k", "--key", help='Your key on the MISP instance')
        self.parser.add_argument("-v", "--verify", action='store_false', help='Disable certificate verification (for self-signed)')
        subparsers = self.parser.add_subparsers(dest='subname')

        # ##### Upload sample to MISP #####
        parser_up = subparsers.add_parser('upload', help='Send malware sample to MISP.',
                                          formatter_class=argparse.RawDescriptionHelpFormatter,
                                          description=textwrap.dedent('''
                                            Distribution levels:
                                                * 0: Your organisation only
                                                * 1: This community only
                                                * 2: Connected communities
                                                * 3: All communities
                                                * 5: Inherit

                                            Sample categories:
                                                * 0: Payload delivery
                                                * 1: Artifacts dropped
                                                * 2: Payload installation
                                                * 3: External analysis

                                            Analysis levels:
                                                * 0: Initial
                                                * 1: Ongoing
                                                * 2: Completed

                                            Threat levels:
                                                * 0: High
                                                * 1: Medium
                                                * 2: Low
                                                * 3: Undefined
                                          '''))
        parser_up.add_argument("-e", "--event", type=int, help="Event ID to update. If None, and you're not connected to a MISP event a new one is created.")
        parser_up.add_argument("-d", "--distrib", type=int, choices=[0, 1, 2, 3, 5], help="Distribution of the attributes for the new event.")
        parser_up.add_argument("-s", "--sharing", type=int, help="Sharing group ID when distribution is set to 4.")
        parser_up.add_argument("-ids", action='store_true', help="Is eligible for automatically creating IDS signatures.")
        parser_up.add_argument("-c", "--categ", type=int, choices=[0, 1, 2, 3], default=1, help="Category of the samples.")
        parser_up.add_argument("-i", "--info", nargs='+', help="Event info field of a new event.")
        parser_up.add_argument("-o", "--comment", nargs='+', help="Comment associated to the sample.")
        parser_up.add_argument("-a", "--analysis", type=int, choices=[0, 1, 2], help="Analysis level a new event.")
        parser_up.add_argument("-t", "--threat", type=int, choices=[0, 1, 2, 3], help="Threat level of a new event.")

        # ##### Download samples from event #####
        parser_down = subparsers.add_parser('download', help='Download malware samples from MISP.')
        group = parser_down.add_mutually_exclusive_group()
        group.add_argument("-e", "--event", type=int, help="Download all the samples related to this event ID.")
        group.add_argument("-l", "--list", nargs='*', help="Download all the samples related to a list of events. Empty list to download all the samples of all the events stored in the current project.")  # noqa
        group.add_argument("--hash", help="Download the sample related to this hash (only MD5).")

        # ##### Search in MISP #####
        parser_search = subparsers.add_parser('search', help='Search in all the attributes.')
        parser_search.add_argument("query", nargs='*', help="String to search (if empty, search the hashes of the current file).")

        # ##### Check hashes on VT #####
        parser_checkhashes = subparsers.add_parser('check_hashes', help='Crosscheck hashes on VT.')
        parser_checkhashes.add_argument("event", nargs='?', default=None, type=int, help="Lookup all the hashes of an event on VT.")
        parser_checkhashes.add_argument("-p", "--populate", action='store_true', help="Automatically populate event with hashes found on VT.")

        # ##### Download Yara rules #####
        parser_checkhashes = subparsers.add_parser('yara', help='Get YARA rules of an event.')
        parser_checkhashes.add_argument("event", nargs='?', default=None, type=int, help="Download the yara rules of that event.")

        # ##### Get Events #####
        parser_pull = subparsers.add_parser('pull', help='Initialize the session with an existing MISP event.')
        parser_pull.add_argument("event", nargs='+', type=int, help="(List of) Event(s) ID.")

        # ##### Create an Event #####
        parser_create_event = subparsers.add_parser('create_event', help='Create a new event on MISP and initialize the session with it.',
                                                    formatter_class=argparse.RawDescriptionHelpFormatter,
                                                    description=textwrap.dedent('''
                                                      Distribution levels:
                                                          * 0: Your organisation only
                                                          * 1: This community only
                                                          * 2: Connected communities
                                                          * 3: All communities
                                                          * 4: Sharing group

                                                      Sharing Group:
                                                          * #: ID of sharing group

                                                      Analysis levels:
                                                          * 0: Initial
                                                          * 1: Ongoing
                                                          * 2: Completed

                                                      Threat levels:
                                                          * 0: High
                                                          * 1: Medium
                                                          * 2: Low
                                                          * 3: Undefined
                                                    '''))
        parser_create_event.add_argument("-d", "--distrib", type=int, choices=[0, 1, 2, 3, 4], help="Distribution of the attributes for the new event.")
        parser_create_event.add_argument("-s", "--sharing", type=int, help="Sharing group ID when distribution is set to 4.")
        parser_create_event.add_argument("-t", "--threat", type=int, choices=[0, 1, 2, 3], help="Threat level of a new event.")
        parser_create_event.add_argument("-a", "--analysis", type=int, choices=[0, 1, 2], help="Analysis level a new event.")
        parser_create_event.add_argument("-i", "--info", required=True, nargs='+', help="Event info field of a new event.")
        parser_create_event.add_argument("--date", help="Date of the event. (Default: today).")

        # ##### Add Hashes #####
        h = subparsers.add_parser("add_hashes", help="If no parameters, add all the hashes of the current session.")
        h.add_argument("-f", "--filename", help="Filename")
        h.add_argument("-m", "--md5", help="MD5")
        h.add_argument("-s", "--sha1", help="SHA1")
        h.add_argument("-a", "--sha256", help="SHA256")

        # ##### Add attributes #####
        parser_add = subparsers.add_parser('add', help='Add attributes to an existing MISP event.')
        subparsers_add = parser_add.add_subparsers(dest='add')
        # Hashes
        # Generic add
        temp_me = MISPEvent()
        if hasattr(temp_me, "types"):
            known_types = temp_me.types
        else:
            # New API
            known_types = temp_me.get_known_types()

        for t in known_types:
            sp = subparsers_add.add_parser(t, help="Add {} to the event.".format(t))
            sp.add_argument(t, nargs='+')

        # ##### Show attributes  #####
        subparsers.add_parser('show', help='Show attributes to an existing MISP event.')

        # ##### Open file #####
        o = subparsers.add_parser('open', help='Open a sample from the temp directory.')
        ox = o.add_mutually_exclusive_group(required=True)
        ox.add_argument("-l", "--list", action='store_true', help="List available files")
        ox.add_argument("-d", "--delete", help="Delete temporary files (use 'all' to remove all the local samples or an Event ID to only remove the associated samples)")
        ox.add_argument("sid", nargs='?', type=int, help='Sample ID to open (from the list option).')

        # ##### Publish an event #####
        subparsers.add_parser('publish', help='Publish an existing MISP event.')

        # ##### Show version #####
        subparsers.add_parser('version', help='Returns the version of the MISP instance.')

        # Store
        s = subparsers.add_parser('store', help='Store the current MISP event in the current project.')
        s.add_argument("-l", "--list", action='store_true', help="List stored MISP events")
        s.add_argument("-u", "--update", action='store_true', help="Update all stored MISP events")
        s.add_argument("-s", "--sync", action='store_true', help="Sync all MISP Events with the remote MISP instance")
        s.add_argument("-d", "--delete", type=int, help="Delete a stored MISP event")
        s.add_argument("-o", "--open", help="Open a stored MISP event")

        # Tags
        s = subparsers.add_parser('tag', help='Tag managment using MISP taxonomies.')
        s.add_argument("-l", "--list", action='store_true', help="List Existing taxonomies.")
        s.add_argument("-d", "--details", help="Display all values of a taxonomy.")
        s.add_argument("-s", "--search", help="Search all tags matching a value.")
        s.add_argument("-e", "--event", help="Add tag to the current event.")
        s.add_argument("-a", "--attribute", nargs='+', help="Add tag to an attribute of the current event. Syntax: <identifier for the attribute> <machinetag>")

        # Galaxies
        s = subparsers.add_parser('galaxies', help='Use misp-galaxy with PyMISPGalaxies.')
        s.add_argument("-l", "--list", action='store_true', help="List existing galaxies.")
        s.add_argument("-d", "--details", help="Display all values of a galaxy.")
        s.add_argument("-v", "--cluster-value", nargs='+', help="Display all details of a cluster value.")
        s.add_argument("-s", "--search", nargs='+', help="Search all galaxies matching a value.")

        # Admin
        s = subparsers.add_parser('admin', help='Administration options.')
        admin_parser = s.add_subparsers(dest='admin')
        # Organisation
        org = admin_parser.add_parser('org', help="Organisation managment.")
        subparsers_org = org.add_subparsers(dest='org')
        # Get
        display = subparsers_org.add_parser('display', help="Display an organisation.")
        display.add_argument('id', help='ID of the organisation to display. Use "local" to display all local organisations, "external" for all remote organisations, and "all", for both.')
        # Search
        search = subparsers_org.add_parser('search', help="Search an organisation by name.")
        search.add_argument('name', help='(Partial) name of the organisation.')
        search.add_argument('-t', '--type', default='local', choices=['local', 'external', 'all'],
                            help='Use "local" to search in all local organisations, "external" for remote organisations, and "all", for both.')
        # Add
        add_org = subparsers_org.add_parser('add', help="Add an organisation.")
        add_org.add_argument('name', help='Organisation name.')
        add_org.add_argument('-u', '--uuid', default=None, help='UUID of the organisation.')
        add_org.add_argument('-d', '--description', default=[], nargs='+', help='Description of the organisation.')
        add_org.add_argument('-t', '--type', default=[], nargs='+', help='Type of the organisation.')
        add_org.add_argument('-n', '--nationality', default=None, help='Nationality of the organisation.')
        add_org.add_argument('-s', '--sector', default=[], nargs='+', help='Sector of the organisation.')
        add_org.add_argument('-c', '--contacts', default=[], nargs='+', help='Contact point(s) in the organisation.')
        add_org.add_argument('--not-local', default=True, action='store_false', help='**Not** a local organisation.')
        # Delete
        delete = subparsers_org.add_parser('delete', help="Delete an organisation.")
        delete.add_argument('id', help='ID of the organisation to delete.')
        # Edit
        edit = subparsers_org.add_parser('edit', help="Edit an organisation.")
        edit.add_argument('id', help='ID of the organisation to edit.')
        edit.add_argument('-n', '--name', help='Organisation name.')
        edit.add_argument('-u', '--uuid', help='UUID of the organisation.')
        edit.add_argument('-d', '--description', default=[], nargs='+', help='Description of the organisation.')
        edit.add_argument('-t', '--type', default=[], nargs='+', help='Type of the organisation.')
        edit.add_argument('--nationality', help='Nationality of the organisation.')
        edit.add_argument('-s', '--sector', default=[], nargs='+', help='Sector of the organisation.')
        edit.add_argument('-c', '--contacts', default=[], nargs='+', help='Contact point(s) in the organisation.')
        edit.add_argument('--not-local', default=True, action='store_false', help='**Not** a local organisation.')

        # User
        user = admin_parser.add_parser('user', help="User managment.")
        subparsers_user = user.add_subparsers(dest='user')
        # Get
        display = subparsers_user.add_parser('display', help="Display a user.")
        display.add_argument('id', help='ID of the user to display. Use "all" to display all users.')
        # Search
        search = subparsers_user.add_parser('search', help="Search a user by email.")
        search.add_argument('name', help='(Partial) email of the user.')
        # Add
        add_usr = subparsers_user.add_parser('add', help="Add a user.")
        add_usr.add_argument('email', help='User email address.')
        add_usr.add_argument('-o', '--org-id', default=None, help='Organisation ID of the user.')
        add_usr.add_argument('-r', '--role-id', default=None, help='Role of the user')
        add_usr.add_argument('-g', '--gpgkey', default=None, help='Path to the GPG public key export')
        add_usr.add_argument('-c', '--change-pw', default=None, action='store_true', help='Force thanging the password after next login')
        add_usr.add_argument('-t', '--termsaccepted', default=None, action='store_true', help='Set the TOC to accepted')
        add_usr.add_argument('-p', '--password', default=None, help='Set a new password')
        add_usr.add_argument('-d', '--disabled', default=None, action='store_true', help='Disable the account')
        # Delete
        delete = subparsers_user.add_parser('delete', help="Delete a user.")
        delete.add_argument('id', help='ID of the user to delete.')
        # Edit
        edit = subparsers_user.add_parser('edit', help="Edit a user.")
        edit.add_argument('id', help='ID of the user to edit.')
        edit.add_argument('-e', '--email', help='User email address.')
        edit.add_argument('-o', '--org-id', default=None, help='Organisation ID of the user.')
        edit.add_argument('-r', '--role-id', default=None, help='Role of the user')
        edit.add_argument('-g', '--gpgkey', default=None, help='Path to the GPG public key export')
        edit.add_argument('-c', '--change-pw', default=None, action='store_true', help='Force thanging the password after next login')
        edit.add_argument('-t', '--termsaccepted', default=None, action='store_true', help='Set the TOC to accepted')
        edit.add_argument('-p', '--password', default=None, help='Set a new password')
        edit.add_argument('-d', '--disabled', default=None, action='store_true', help='Disable the account')

        # Role
        role = admin_parser.add_parser('role', help="Role managment.")
        subparsers_role = role.add_subparsers(dest='role')
        # Get
        display = subparsers_role.add_parser('display', help="Display all the roles.")
        # Search
        search = subparsers_role.add_parser('search', help="Search a role by name.")
        search.add_argument('name', help='(Partial) name of the role.')

        # Tags
        t = admin_parser.add_parser('tag', help="Tag managment.")
        subparsers_tag = t.add_subparsers(dest='tag')
        # Get
        display = subparsers_tag.add_parser('display', help="Display all the tags.")
        # Search
        search = subparsers_tag.add_parser('search', help="Search a tag by name.")
        search.add_argument('name', help='(Partial) name of the tag.')

        self.categories = {0: 'Payload delivery', 1: 'Artifacts dropped', 2: 'Payload installation', 3: 'External analysis'}
Beispiel #32
0
        "The distribution setting used for the attributes and for the newly created event, if relevant. [0-3]."
    )
    parser.add_argument(
        "-i",
        "--info",
        help="Used to populate the event info field if no event ID supplied.")
    parser.add_argument(
        "-a",
        "--analysis",
        type=int,
        help=
        "The analysis level of the newly created event, if applicable. [0-2]")
    parser.add_argument(
        "-t",
        "--threat",
        type=int,
        help=
        "The threat level ID of the newly created event, if applicable. [1-4]")
    args = parser.parse_args()

    misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)

    event = MISPEvent()
    event.distribution = args.distrib
    event.threat_level_id = args.threat
    event.analysis = args.analysis
    event.info = args.info

    event = misp.add_event(event, pythonify=True)
    print(event)
Beispiel #33
0
 def setUp(self):
     self.maxDiff = None
     self.mispevent = MISPEvent()
Beispiel #34
0
 def __init__(self):
     self.misp_event = MISPEvent()
     self.event = defaultdict(dict)
     self.misp_event['Galaxy'] = []
Beispiel #35
0
def twitter_account(data):
    """
    Do the things.
    :param data:
    :return:
    """
    misp_event_id = data['misp_event_id']
    twitter_post_id = data['twitter_post_id']
    response_url = data['response_url']

    try:
        # Get the Twitter status.
        status = twitter_get_account(twitter_post_id)

        # Extract relevant values from the Twitter status.
        microblog_data = transform_twitter_account(status._json)

        # Load the microblog version from it's definition.json file.
        with open("misp-objects/twitter-account/definition.json") as f:
            microblog_definition = json.load(f)
            f.close()

        # Create the MISP mircroblog object.
        # TODO: get the object definition from github
        # misp_objects_path_custom searches the var path for {objectname}/definition.json to load the object definition
        # This file needs to be updated when the upstream object is updated.
        microblog = TwitterAccountObject(
            parameters=microblog_data,
            misp_objects_path_custom="misp-objects",
            template_version=str(microblog_definition["version"]))

        # Get the MISP event.
        working_event = misp.get_event(misp_event_id,
                                       extended=True,
                                       pythonify=True)

        # Get the Slackbot's MISP org ID.
        user_profile = misp.get_user("me")
        bot_org_id = user_profile["User"]["org_id"]

        # If the bot org can update the MISP Event with the new microblog do so.
        if str(bot_org_id) == str(working_event["org_id"]):
            working_event.Object.append(microblog)
            result = misp.update_event(working_event)
            print(result)
        else:
            new_event = True
            # If an extension exists for Slackbot objects use it.
            if "extensionEvents" in working_event:
                for k, event_extension in working_event[
                        "extensionEvents"].items():
                    if event_extension["Orgc"]["id"] == bot_org_id:
                        if event_extension[
                                "info"] == "Covid Slack: Disinfo Bot":
                            extension_event = misp.get_event(
                                event_extension["id"], pythonify=True)
                            extension_event.Object.append(microblog)
                            result = misp.update_event(extension_event)
                            print(result)
                            new_event = False
            # Create a new extension to the parent event.
            if new_event:
                extended_event = MISPEvent()
                extended_event.info = "Covid Slack: Disinfo Bot"
                extended_event.extends_uuid = working_event["id"]
                extended_event.Object.append(microblog)
                result = misp.add_event(extended_event, pythonify=True)
                print(result)

        # Build slack response.
        response = {'blocks': [], 'response_type': 'in_channel'}
        response['blocks'].append({
            'type': 'section',
            'text': {
                'type': 'mrkdwn',
                'text': 'Twitter Status: {}'.format(twitter_post_id)
            }
        })

        twitter_message = ""
        if microblog_data.get('name'):
            twitter_message += 'Username: {}\n'.format(microblog_data['name'])
        if microblog_data.get('display-name'):
            twitter_message += 'Display Name: {}\n'.format(
                microblog_data['display-name'])
        if microblog_data.get('verified'):
            twitter_message += 'Verified Account: {}\n'.format(
                microblog_data['verified'])
        if microblog_data.get('description'):
            twitter_message += 'Bio: {}\n'.format(
                microblog_data['description'])

        if len(microblog_data['hashtag']) > 0:
            twitter_message += 'Hashtags:\n'
            for hashtag in microblog_data['hashtag']:
                twitter_message += '* {}\n'.format(hashtag)

        if len(microblog_data['embedded-link']) > 0:
            twitter_message += 'Embedded URLs:\n'
            for url in microblog_data['embedded-link']:
                twitter_message += '* {}\n'.format(url)

        # Add Twitter message as block.
        response['blocks'].append({
            'type': 'section',
            'text': {
                'type': 'mrkdwn',
                'text': twitter_message
            }
        })

        requests.post(response_url, json=response)
    except Exception:
        logger.info(traceback.print_exc(file=sys.stdout))
        message = "An error has occurred!"
        resp = build_response(message, False)
        requests.post(response_url, json=resp)
Beispiel #36
0
 def submit_tf_update(misp: ExpandedPyMISP, attributes: list) -> MISPEvent:
     """
         create/update abuse.ch MISP-Event and append the new attributes
     """
     eventinfo = event_info_template.format(
         datetime.now().strftime(info_dateformat))
     # logging.debug(eventinfo)
     events = misp.search(controller='events',
                          eventinfo=eventinfo,
                          org=1,
                          pythonify=True)
     if events:  # current event exists already
         event = events[0]
     else:  # create a new event
         event = MISPEvent()
         event.distribution = event_distribution
         event.threat_level_id = event_threat_level
         event.analysis = 2
         event.info = eventinfo
         for tag in tagging:
             event.add_tag(tag)
         event = misp.add_event(event, pythonify=True)
     for att in attributes:
         event.add_attribute(**att)
     event.published = autopublish
     return misp.update_event(event)
Beispiel #37
0
 def load_events_directory(self, directory):
     self.events = []
     for path in glob.glob(os.path.join(directory, '*.json')):
         e = MISPEvent()
         e.load(path)
         self.import_event(e)
    "reportname":"top-attacks",
    "key":"#{API_KEY}"  # put your Panorama api key here
}

# try with "..., verify=False)" if you get an SSL error
response = requests.request("GET", url, params=querystring)

resp_text = response.text

json_data = json.loads(json.dumps(xmltodict.parse(resp_text)))

# initialize and set MISPOrganisation
orgc = MISPOrganisation()
orgc.name = 'Palo Alto'
orgc.id = '#{ORGC_ID}'  # organisation id
orgc.uuid = '#{ORGC_UUID}'  # organisation uuid
# initialize and set MISPEvent()
event = MISPEvent()
event.Orgc = orgc
event.info = json_data['report']['result']['@name'] + " | " + json_data['report']['result']['@range']
event.distribution = 0  # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 0  # Optional, defaults to MISP.default_event_threat_level in MISP config
event.analysis = 0  # Optional, defaults to 0 (initial analysis)
event.add_tag('firewall threats')

for threatid in json_data['report']['result']['entry']:
    attribute = event.add_attribute('comment', threatid['threatid'])
    attribute.comment = threatid['count']

misp.add_event(event.to_json())
Beispiel #39
0
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from pymisp import ExpandedPyMISP, MISPEvent
from pymisp import MISPObject
from keys import misp_url, misp_key, misp_verifycert
from datetime import date

misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)

event = MISPEvent()
event.info = 'IoT malware'  # Event Title
event.distribution = 1  # 0 = Your Organisation Only, 1 = Community
event.threat_level_id = 2  # 1 = High, 2 = Medium, 3 = Low
event.analysis = 2  # 0 (initial analysis), 1 (On-Going), 2 (Complete)

event.add_tag('malware_classification:malware-category="Botnet"')
event.add_tag('tlp:amber')

d = date.today()
event.set_date(d)

attribute_second = event.add_attribute('url',
                                       'http://1.2.3.4/example',
                                       disable_correlation=False,
                                       comment="Botnet example text",
                                       to_ids=False)

event = misp.add_event(event, pythonify=True)

# Publish event
Beispiel #40
0
class TestMISPEvent(unittest.TestCase):
    def setUp(self):
        self.maxDiff = None
        self.mispevent = MISPEvent()

    def init_event(self):
        self.mispevent.info = 'This is a test'
        self.mispevent.distribution = 1
        self.mispevent.threat_level_id = 1
        self.mispevent.analysis = 1
        self.mispevent.set_date("2017-12-31")  # test the set date method

    def test_simple(self):
        with open('tests/mispevent_testfiles/simple.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_event(self):
        self.init_event()
        self.mispevent.publish()
        with open('tests/mispevent_testfiles/event.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_loadfile(self):
        self.mispevent.load_file('tests/mispevent_testfiles/event.json')
        with open('tests/mispevent_testfiles/event.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_event_tag(self):
        self.init_event()
        self.mispevent.add_tag('bar')
        self.mispevent.add_tag(name='baz')
        new_tag = MISPTag()
        new_tag.from_dict(name='foo')
        self.mispevent.add_tag(new_tag)
        with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_attribute(self):
        self.init_event()
        a = self.mispevent.add_attribute('filename', 'bar.exe')
        del a.uuid
        a = self.mispevent.add_attribute_tag('osint', 'bar.exe')
        attr_tags = self.mispevent.get_attribute_tag('bar.exe')
        self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint')
        self.assertEqual(attr_tags[0].name, 'osint')
        with open('tests/mispevent_testfiles/attribute.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))
        # Fake setting an attribute ID for testing
        self.mispevent.attributes[0].id = 42
        self.mispevent.delete_attribute(42)
        with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_object_tag(self):
        self.mispevent.add_object(name='file', strict=True)
        a = self.mispevent.objects[0].add_attribute('filename', value='')
        self.assertEqual(a, None)
        a = self.mispevent.objects[0].add_attribute('filename', value=None)
        self.assertEqual(a, None)
        a = self.mispevent.objects[0].add_attribute('filename',
                                                    value='bar',
                                                    Tag=[{
                                                        'name': 'blah'
                                                    }])
        del a.uuid
        self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name,
                         'blah')
        self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(
            ['filename']))
        self.assertEqual(
            len(self.mispevent.objects[0].get_attributes_by_relation(
                'filename')), 1)
        self.mispevent.add_object(name='url', strict=True)
        a = self.mispevent.objects[1].add_attribute(
            'url', value='https://www.circl.lu')
        del a.uuid
        self.mispevent.objects[0].uuid = 'a'
        self.mispevent.objects[1].uuid = 'b'
        reference = self.mispevent.objects[0].add_reference(
            self.mispevent.objects[1], 'baz', comment='foo')
        del reference.uuid
        self.assertEqual(
            self.mispevent.objects[0].references[0].relationship_type, 'baz')
        with open('tests/mispevent_testfiles/event_obj_attr_tag.json',
                  'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    @unittest.skip(
        "Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168"
    )
    def test_object_level_tag(self):
        self.mispevent.add_object(name='file', strict=True)
        self.mispevent.objects[0].add_attribute('filename', value='bar')
        self.mispevent.objects[0].add_tag('osint')
        self.mispevent.objects[0].uuid = 'a'
        with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_malware(self):
        with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
            pseudofile = BytesIO(f.read())
        self.init_event()
        a = self.mispevent.add_attribute('malware-sample',
                                         'bar.exe',
                                         data=pseudofile)
        del a.uuid
        attribute = self.mispevent.attributes[0]
        self.assertEqual(attribute.malware_binary, pseudofile)
        with open('tests/mispevent_testfiles/malware.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_existing_malware(self):
        self.mispevent.load_file(
            'tests/mispevent_testfiles/malware_exist.json')
        with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
            pseudofile = BytesIO(f.read())
        self.assertEqual(
            self.mispevent.objects[0].get_attributes_by_relation(
                'malware-sample')[0].malware_binary.read(), pseudofile.read())

    def test_sighting(self):
        sighting = MISPSighting()
        sighting.from_dict(value='1', type='bar', timestamp=11111111)
        with open('tests/mispevent_testfiles/sighting.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(sighting.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_existing_event(self):
        self.mispevent.load_file(
            'tests/mispevent_testfiles/existing_event.json')
        with open('tests/mispevent_testfiles/existing_event.json', 'r') as f:
            ref_json = json.load(f)

        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_shadow_attributes_existing(self):
        self.mispevent.load_file('tests/mispevent_testfiles/shadow.json')
        with open('tests/mispevent_testfiles/shadow.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    @unittest.skip("Not supported on MISP.")
    def test_shadow_attributes(self):
        self.init_event()
        p = self.mispevent.add_proposal(type='filename', value='baz.jpg')
        del p.uuid
        a = self.mispevent.add_attribute('filename', 'bar.exe')
        del a.uuid
        p = self.mispevent.attributes[0].add_proposal(type='filename',
                                                      value='bar.pdf')
        del p.uuid
        with open('tests/mispevent_testfiles/proposals.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_default_attributes(self):
        self.mispevent.add_object(name='file', strict=True)
        a = self.mispevent.objects[0].add_attribute('filename',
                                                    value='bar',
                                                    Tag=[{
                                                        'name': 'blah'
                                                    }])
        del a.uuid
        a = self.mispevent.objects[0].add_attribute('pattern-in-file',
                                                    value='baz')
        self.assertEqual(a.category, 'Artifacts dropped')
        del a.uuid
        self.mispevent.add_object(name='file',
                                  strict=False,
                                  default_attributes_parameters=self.mispevent.
                                  objects[0].attributes[0])
        a = self.mispevent.objects[1].add_attribute('filename', value='baz')
        del a.uuid
        self.mispevent.objects[0].uuid = 'a'
        self.mispevent.objects[1].uuid = 'b'
        with open('tests/mispevent_testfiles/event_obj_def_param.json',
                  'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_obj_default_values(self):
        self.init_event()
        self.mispevent.add_object(name='whois', strict=True)
        a = self.mispevent.objects[0].add_attribute(
            'registrar', value='registar.example.com')
        del a.uuid
        a = self.mispevent.objects[0].add_attribute('domain',
                                                    value='domain.example.com')
        del a.uuid
        a = self.mispevent.objects[0].add_attribute('nameserver',
                                                    value='ns1.example.com')
        del a.uuid
        a = self.mispevent.objects[0].add_attribute(
            'nameserver',
            value='ns2.example.com',
            disable_correlation=False,
            to_ids=True,
            category='External analysis')
        del a.uuid
        self.mispevent.objects[0].uuid = 'a'
        with open('tests/mispevent_testfiles/def_param.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_event_not_edited(self):
        self.mispevent.load_file(
            'tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)

    def test_event_edited(self):
        self.mispevent.load_file(
            'tests/mispevent_testfiles/existing_event.json')
        self.mispevent.info = 'blah'
        self.assertTrue(self.mispevent.edited)

    def test_event_tag_edited(self):
        self.mispevent.load_file(
            'tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)
        self.mispevent.add_tag('foo')
        self.assertTrue(self.mispevent.edited)

    def test_event_attribute_edited(self):
        self.mispevent.load_file(
            'tests/mispevent_testfiles/existing_event.json')
        self.mispevent.attributes[0].value = 'blah'
        self.assertTrue(self.mispevent.attributes[0].edited)
        self.assertFalse(self.mispevent.attributes[1].edited)
        self.assertTrue(self.mispevent.edited)

    def test_event_attribute_tag_edited(self):
        self.mispevent.load_file(
            'tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)
        self.mispevent.attributes[0].tags[0].name = 'blah'
        self.assertTrue(self.mispevent.attributes[0].tags[0].edited)
        self.assertFalse(self.mispevent.attributes[0].tags[1].edited)
        self.assertTrue(self.mispevent.attributes[0].edited)
        self.assertTrue(self.mispevent.edited)

    def test_event_attribute_tag_edited_second(self):
        self.mispevent.load_file(
            'tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)
        self.mispevent.attributes[0].add_tag(name='blah')
        self.assertTrue(self.mispevent.attributes[0].edited)
        self.assertTrue(self.mispevent.edited)

    def test_event_object_edited(self):
        self.mispevent.load_file(
            'tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)
        self.mispevent.objects[0].comment = 'blah'
        self.assertTrue(self.mispevent.objects[0].edited)
        self.assertFalse(self.mispevent.objects[1].edited)
        self.assertTrue(self.mispevent.edited)

    def test_event_object_attribute_edited(self):
        self.mispevent.load_file(
            'tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)
        self.mispevent.objects[0].attributes[0].comment = 'blah'
        self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
        self.assertTrue(self.mispevent.objects[0].edited)
        self.assertTrue(self.mispevent.edited)

    def test_event_object_attribute_edited_tag(self):
        self.mispevent.load_file(
            'tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)
        self.mispevent.objects[0].attributes[0].add_tag('blah')
        self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
        self.assertTrue(self.mispevent.objects[0].edited)
        self.assertTrue(self.mispevent.edited)
        with open('tests/mispevent_testfiles/existing_event_edited.json',
                  'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))

    def test_obj_by_id(self):
        self.mispevent.load_file(
            'tests/mispevent_testfiles/existing_event.json')
        misp_obj = self.mispevent.get_object_by_id(1556)
        self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f')

    def test_userdefined_object(self):
        self.init_event()
        self.mispevent.add_object(
            name='test_object_template',
            strict=True,
            misp_objects_path_custom='tests/mispevent_testfiles')
        with self.assertRaises(InvalidMISPObject) as e:
            # Fail on required
            self.mispevent.to_json(sort_keys=True, indent=2)
        if sys.version_info >= (3, ):
            self.assertEqual(e.exception.message,
                             '{\'member3\'} are required.')
        else:
            # Python2 bullshit
            self.assertEqual(e.exception.message,
                             'set([u\'member3\']) are required.')

        a = self.mispevent.objects[0].add_attribute('member3', value='foo')
        del a.uuid
        with self.assertRaises(InvalidMISPObject) as e:
            # Fail on requiredOneOf
            self.mispevent.to_json(sort_keys=True, indent=2)
        self.assertEqual(
            e.exception.message,
            'At least one of the following attributes is required: member1, member2'
        )

        a = self.mispevent.objects[0].add_attribute('member1', value='bar')
        del a.uuid
        a = self.mispevent.objects[0].add_attribute('member1', value='baz')
        del a.uuid
        with self.assertRaises(InvalidMISPObject) as e:
            # member1 is not a multiple
            self.mispevent.to_json(sort_keys=True, indent=2)
        self.assertEqual(e.exception.message,
                         'Multiple occurrences of member1 is not allowed')

        self.mispevent.objects[0].attributes = self.mispevent.objects[
            0].attributes[:2]
        self.mispevent.objects[0].uuid = 'a'
        with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(sort_keys=True, indent=2),
                         json.dumps(ref_json, sort_keys=True, indent=2))
Beispiel #41
0
 def handle_ipaddress(self, ipaddress, event: pymisp.MISPEvent):
     """Handle a single IP address."""
     event.add_attribute("ip-dst", str(ipaddress))
     return event
Beispiel #42
0
def check_hashes(self):
    if self.offline_mode:
        self.log('error', 'Offline mode, unable to query VirusTotal')
        return
    event_id = self._get_eventid()
    if event_id is None:
        return
    event = self.misp.get(event_id)
    if self._has_error_message(event):
        return

    misp_event = MISPEvent()
    misp_event.load(event)
    hashes_to_expand = {}
    hashes_expanded = []  # Thoses hashes are known and already processed
    local_samples_hashes = []
    partial_objects = {}
    for o in misp_event.Object:
        if o.name != 'file':
            continue
        if o.has_attributes_by_relation(['md5', 'sha1', 'sha256']):
            # This object has all the hashes we care about
            tmphashes = []
            tmphashes += [h.value for h in o.get_attributes_by_relation('md5')]
            tmphashes += [h.value for h in o.get_attributes_by_relation('sha1')]
            tmphashes += [h.value for h in o.get_attributes_by_relation('sha256')]
            # Make sure to query VT for the sha256, even if expanded locally
            hashes_to_expand[o.get_attributes_by_relation('sha256')[0].value] = o.get_attributes_by_relation('sha256')[0]
            if o.has_attributes_by_relation(['malware-sample']):
                # ... and it has a malware sample
                local_samples_hashes += tmphashes
            hashes_expanded += tmphashes
        elif o.has_attributes_by_relation(['malware-sample']):
            # This object has a malware sample, but is missing hashes. We can expand locally.
            # get the MD5 from the malware-sample attribute
            malware_sample = o.get_attributes_by_relation('malware-sample')[0]  # at most one sample/file object
            local_samples_hashes.append(malware_sample.value.split('|')[1])
            local_samples_hashes += [h.value for h in o.get_attributes_by_relation('md5')]
            local_samples_hashes += [h.value for h in o.get_attributes_by_relation('sha1')]
            local_samples_hashes += [h.value for h in o.get_attributes_by_relation('sha256')]
            if self.args.populate:
                # The object is missing hashes, keeping track of it for expansion if it isn't already done.
                partial_objects[o.uuid] = malware_sample

        else:
            sha256 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('sha256')}
            sha1 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('sha1')}
            md5 = {attribute.value: attribute for attribute in o.get_attributes_by_relation('md5')}
            if sha256:
                hashes_to_expand.update(sha256)
            elif sha1:
                hashes_to_expand.update(sha1)
            elif md5:
                hashes_to_expand.update(md5)

    for ref_uuid, sample in partial_objects.items():
        if sample.value.split('|')[1] in hashes_expanded:
            # Already expanded in an other object
            continue
        new_obj, hashes = self._expand_local_sample(pseudofile=sample.malware_binary,
                                                    filename=sample.value.split('|')[0],
                                                    refobj=ref_uuid,
                                                    default_attributes_paramaters=sample)
        misp_event.Object += new_obj
        local_samples_hashes += hashes
        # Make sure to query VT for the sha256, even if expanded locally
        hashes_to_expand[hashes[0]] = sample

    hashes_expanded += local_samples_hashes
    for a in misp_event.attributes:
        if a.type == 'malware-sample' and a.value.split('|')[1] not in hashes_expanded:
            new_obj, hashes = self._expand_local_sample(pseudofile=a.malware_binary,
                                                        filename=a.value.split('|')[0],
                                                        default_attributes_paramaters=a)
            misp_event.Object += new_obj
            local_samples_hashes += hashes
            # Make sure to query VT for the sha256, even if expanded locally
            hashes_to_expand[hashes[0]] = a
        elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256'):
            # We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway
            fname, hashval = a.value.split('|')
            hashes_to_expand[hashval] = a
        elif a.type in ('md5', 'sha1', 'sha256'):
            # We don't care if the hashes are in hashes_expanded or hashes_to_expand: they are firtered out later anyway
            hashes_to_expand[a.value] = a

    unk_vt_hashes = []
    if cfg.virustotal.virustotal_has_private_key is False:
        quota = 4
        timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)

    hashes_expanded += local_samples_hashes
    processed_on_vt = []
    # Make sure to start getting reports for the longest possible hashes (reduce risks of collisions)
    for to_expand in sorted(list(set(hashes_to_expand)), key=len):
        if to_expand in processed_on_vt:
            # Always run VT, once per sample
            continue
        original_attribute = hashes_to_expand[to_expand]
        if original_attribute.get('object_id'):
            original_object_id = original_attribute.get('object_id')
        vt_object = self._make_VT_object(to_expand, original_attribute)
        if not vt_object:
            unk_vt_hashes.append(to_expand)
            continue
        result = vt_object.get_report()
        md5 = result['md5']
        sha1 = result['sha1']
        sha256 = result['sha256']
        processed_on_vt += [sha256, sha1, md5]
        if all(h in local_samples_hashes for h in [md5, sha1, sha256]):
            self.log('success', 'Sample available in MISP:')
        else:
            self.log('success', 'Sample available in VT:')
        self.log('item', '{}\n\t{}\n\t{}\n\t{}'.format(result["permalink"], md5, sha1, sha256))
        if self.args.populate:
            if not all(h in hashes_expanded for h in [md5, sha1, sha256]):
                # If all the "new" expanded hashes are in the hashes_expanded list, skip
                file_object = MISPObject('file', default_attributes_paramaters=original_attribute)
                file_object.add_attribute('md5', value=md5)
                file_object.add_attribute('sha1', value=sha1)
                file_object.add_attribute('sha256', value=sha256)
                file_object.add_reference(vt_object.uuid, 'analysed-with')
                misp_event.Object.append(file_object)
                hashes_expanded += [md5, sha1, sha256]
            else:
                if not original_object_id or original_object_id == '0':
                    # Not an object, but the hashes are in an other object, skipping
                    continue
                else:
                    # We already have a MISP object, adding the link to the new VT object
                    file_object = misp_event.get_object_by_id(original_object_id)
                    file_object.add_reference(vt_object.uuid, 'analysed-with')
            misp_event.Object.append(vt_object)

        if cfg.virustotal.virustotal_has_private_key is False:
            if quota > 0:
                quota -= 1
            else:
                waiting_time = (timeout - datetime.datetime.now()).seconds
                if waiting_time > 0:
                    self.log('warning', 'No private API key, 4 queries/min is the limit. Waiting for {} seconds.'.format(waiting_time))
                    time.sleep(waiting_time)
                quota = 4
                timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)

    if self.args.populate:
        self._populate(misp_event)
    if len(unk_vt_hashes) > 0:
        self.log('error', 'Unknown on VT:')
        for h in unk_vt_hashes:
            self.log('item', '{}'.format(h))
Beispiel #43
0
 def handle_yarasignature(self, yarasignature, event: pymisp.MISPEvent):
     """Handle a single YARA signature."""
     event.add_attribute("yara", str(yarasignature))
     return event
Beispiel #44
0
def create_event(misp):
    event = MISPEvent()
    event.distribution = 0
    event.threat_level_id = 1
    event.analysis = 0
    return event
Beispiel #45
0
    def __init__(self, attribute_props: dict):
        self.event = MISPEvent()
        self.enrichment_object = MISPObject("Recorded Future Enrichment")
        description = ("An object containing the enriched attribute and "
                       "related entities from Recorded Future.")
        self.enrichment_object.from_dict(**{
            "meta-category": "misc",
            "description": description,
            "distribution": 0
        })

        # Create a copy of enriched attribute to add tags to
        temp_attr = MISPAttribute()
        temp_attr.from_dict(**attribute_props)
        self.enriched_attribute = MISPAttribute()
        self.enriched_attribute.from_dict(**{
            "value": temp_attr.value,
            "type": temp_attr.type,
            "distribution": 0
        })

        self.related_attributes: List[Tuple[str, MISPAttribute]] = []
        self.color_picker = RFColors()
        self.galaxy_finder = GalaxyFinder()

        # Mapping from MISP-type to RF-type
        self.type_to_rf_category = {
            "ip": "ip",
            "ip-src": "ip",
            "ip-dst": "ip",
            "ip-src|port": "ip",
            "ip-dst|port": "ip",
            "domain": "domain",
            "hostname": "domain",
            "md5": "hash",
            "sha1": "hash",
            "sha256": "hash",
            "uri": "url",
            "url": "url",
            "vulnerability": "vulnerability",
            "weakness": "vulnerability",
        }

        # Related entities have 'Related' as part of the word and Links entities from RF
        # portrayed as related attributes in MISP
        self.related_attribute_types = [
            "RelatedIpAddress",
            "RelatedInternetDomainName",
            "RelatedHash",
            "RelatedEmailAddress",
            "RelatedCyberVulnerability",
            "IpAddress",
            "InternetDomainName",
            "Hash",
            "EmailAddress",
            "CyberVulnerability",
        ]
        # Related entities have 'Related' as part of the word and and Links entities from RF portrayed as tags in MISP
        self.galaxy_tag_types = [
            "RelatedMalware",
            "RelatedThreatActor",
            "Threat Actor",
            "MitreAttackIdentifier",
            "Malware",
        ]
Beispiel #46
0
class VirusTotalParser(object):
    def __init__(self, apikey, limit):
        self.apikey = apikey
        self.limit = limit
        self.base_url = "https://www.virustotal.com/vtapi/v2/{}/report"
        self.misp_event = MISPEvent()
        self.parsed_objects = {}
        self.input_types_mapping = {
            'ip-src': self.parse_ip,
            'ip-dst': self.parse_ip,
            'domain': self.parse_domain,
            'hostname': self.parse_domain,
            'md5': self.parse_hash,
            'sha1': self.parse_hash,
            'sha256': self.parse_hash,
            'url': self.parse_url
        }
        self.proxies = None

    def query_api(self, attribute):
        self.attribute = MISPAttribute()
        self.attribute.from_dict(**attribute)
        return self.input_types_mapping[self.attribute.type](
            self.attribute.value, recurse=True)

    def get_result(self):
        event = json.loads(self.misp_event.to_json())
        results = {
            key: event[key]
            for key in ('Attribute', 'Object') if (key in event and event[key])
        }
        return {'results': results}

    ################################################################################
    ####                         Main parsing functions                         #### # noqa
    ################################################################################

    def parse_domain(self, domain, recurse=False):
        req = requests.get(self.base_url.format('domain'),
                           params={
                               'apikey': self.apikey,
                               'domain': domain
                           },
                           proxies=self.proxies)
        if req.status_code != 200:
            return req.status_code
        req = req.json()
        hash_type = 'sha256'
        whois = 'whois'
        feature_types = {
            'communicating': 'communicates-with',
            'downloaded': 'downloaded-from',
            'referrer': 'referring'
        }
        siblings = (self.parse_siblings(domain)
                    for domain in req['domain_siblings'])
        uuid = self.parse_resolutions(
            req['resolutions'],
            req['subdomains'] if 'subdomains' in req else None, siblings)
        for feature_type, relationship in feature_types.items():
            for feature in ('undetected_{}_samples', 'detected_{}_samples'):
                for sample in req.get(feature.format(feature_type),
                                      [])[:self.limit]:
                    status_code = self.parse_hash(sample[hash_type], False,
                                                  uuid, relationship)
                    if status_code != 200:
                        return status_code
        if req.get(whois):
            whois_object = MISPObject(whois)
            whois_object.add_attribute('text', type='text', value=req[whois])
            self.misp_event.add_object(**whois_object)
        return self.parse_related_urls(req, recurse, uuid)

    def parse_hash(self, sample, recurse=False, uuid=None, relationship=None):
        req = requests.get(self.base_url.format('file'),
                           params={
                               'apikey': self.apikey,
                               'resource': sample
                           },
                           proxies=self.proxies)
        status_code = req.status_code
        if req.status_code == 200:
            req = req.json()
            vt_uuid = self.parse_vt_object(req)
            file_attributes = []
            for hash_type in ('md5', 'sha1', 'sha256'):
                if req.get(hash_type):
                    file_attributes.append({
                        'type': hash_type,
                        'object_relation': hash_type,
                        'value': req[hash_type]
                    })
            if file_attributes:
                file_object = MISPObject('file')
                for attribute in file_attributes:
                    file_object.add_attribute(**attribute)
                file_object.add_reference(vt_uuid, 'analyzed-with')
                if uuid and relationship:
                    file_object.add_reference(uuid, relationship)
                self.misp_event.add_object(**file_object)
        return status_code

    def parse_ip(self, ip, recurse=False):
        req = requests.get(self.base_url.format('ip-address'),
                           params={
                               'apikey': self.apikey,
                               'ip': ip
                           },
                           proxies=self.proxies)
        if req.status_code != 200:
            return req.status_code
        req = req.json()
        if req.get('asn'):
            asn_mapping = {
                'network': ('ip-src', 'subnet-announced'),
                'country': ('text', 'country')
            }
            asn_object = MISPObject('asn')
            asn_object.add_attribute('asn', type='AS', value=req['asn'])
            for key, value in asn_mapping.items():
                if req.get(key):
                    attribute_type, relation = value
                    asn_object.add_attribute(relation,
                                             type=attribute_type,
                                             value=req[key])
            self.misp_event.add_object(**asn_object)
        uuid = self.parse_resolutions(
            req['resolutions']) if req.get('resolutions') else None
        return self.parse_related_urls(req, recurse, uuid)

    def parse_url(self, url, recurse=False, uuid=None):
        req = requests.get(self.base_url.format('url'),
                           params={
                               'apikey': self.apikey,
                               'resource': url
                           },
                           proxies=self.proxies)
        status_code = req.status_code
        if req.status_code == 200:
            req = req.json()
            vt_uuid = self.parse_vt_object(req)
            if not recurse:
                feature = 'url'
                url_object = MISPObject(feature)
                url_object.add_attribute(feature, type=feature, value=url)
                url_object.add_reference(vt_uuid, 'analyzed-with')
                if uuid:
                    url_object.add_reference(uuid, 'hosted-in')
                self.misp_event.add_object(**url_object)
        return status_code

    ################################################################################
    ####                      Additional parsing functions                      #### # noqa
    ################################################################################

    def parse_related_urls(self, query_result, recurse, uuid=None):
        if recurse:
            for feature in ('detected_urls', 'undetected_urls'):
                if feature in query_result:
                    for url in query_result[feature]:
                        value = url['url'] if isinstance(url, dict) else url[0]
                        status_code = self.parse_url(value, False, uuid)
                        if status_code != 200:
                            return status_code
        else:
            for feature in ('detected_urls', 'undetected_urls'):
                if feature in query_result:
                    for url in query_result[feature]:
                        value = url['url'] if isinstance(url, dict) else url[0]
                        self.misp_event.add_attribute('url', value)
        return 200

    def parse_resolutions(self, resolutions, subdomains=None, uuids=None):
        domain_ip_object = MISPObject('domain-ip')
        if self.attribute.type in ('domain', 'hostname'):
            domain_ip_object.add_attribute('domain',
                                           type='domain',
                                           value=self.attribute.value)
            attribute_type, relation, key = ('ip-dst', 'ip', 'ip_address')
        else:
            domain_ip_object.add_attribute('ip',
                                           type='ip-dst',
                                           value=self.attribute.value)
            attribute_type, relation, key = ('domain', 'domain', 'hostname')
        for resolution in resolutions:
            domain_ip_object.add_attribute(relation,
                                           type=attribute_type,
                                           value=resolution[key])
        if subdomains:
            for subdomain in subdomains:
                attribute = MISPAttribute()
                attribute.from_dict(**dict(type='domain', value=subdomain))
                self.misp_event.add_attribute(**attribute)
                domain_ip_object.add_reference(attribute.uuid, 'subdomain')
        if uuids:
            for uuid in uuids:
                domain_ip_object.add_reference(uuid, 'sibling-of')
        self.misp_event.add_object(**domain_ip_object)
        return domain_ip_object.uuid

    def parse_siblings(self, domain):
        attribute = MISPAttribute()
        attribute.from_dict(**dict(type='domain', value=domain))
        self.misp_event.add_attribute(**attribute)
        return attribute.uuid

    def parse_vt_object(self, query_result):
        if query_result['response_code'] == 1:
            vt_object = MISPObject('virustotal-report')
            vt_object.add_attribute('permalink',
                                    type='link',
                                    value=query_result['permalink'])
            detection_ratio = '{}/{}'.format(query_result['positives'],
                                             query_result['total'])
            vt_object.add_attribute('detection-ratio',
                                    type='text',
                                    value=detection_ratio,
                                    disable_correlation=True)
            self.misp_event.add_object(**vt_object)
            return vt_object.uuid

    def set_proxy_settings(self, config: dict) -> dict:
        """Returns proxy settings in the requests format.
        If no proxy settings are set, return None."""
        proxies = None
        host = config.get('proxy_host')
        port = config.get('proxy_port')
        username = config.get('proxy_username')
        password = config.get('proxy_password')

        if host:
            if not port:
                misperrors['error'] = 'The virustotal_proxy_host config is set, ' \
                                    'please also set the virustotal_proxy_port.'
                raise KeyError
            parsed = urlparse(host)
            if 'http' in parsed.scheme:
                scheme = 'http'
            else:
                scheme = parsed.scheme
            netloc = parsed.netloc
            host = f'{netloc}:{port}'

            if username:
                if not password:
                    misperrors['error'] = 'The virustotal_proxy_username config is set, ' \
                                        'please also set the virustotal_proxy_password.'
                    raise KeyError
                auth = f'{username}:{password}'
                host = auth + '@' + host

            proxies = {
                'http': f'{scheme}://{host}',
                'https': f'{scheme}://{host}'
            }
        self.proxies = proxies
        return True
Beispiel #47
0
    def add_event(self):
        try:
            event = MISPEvent()
            event.distribution = 0

            # ATD Threat mapping to MISP Threat Level
            atd_threat_level = self.query['Summary']['Verdict']['Severity']
            if not atd_threat_level:
                pass
            else:
                if atd_threat_level == '3':
                    event.threat_level_id = 1
                elif atd_threat_level == '4':
                    event.threat_level_id = 2
                elif atd_threat_level == '5':
                    event.threat_level_id = 3
                else:
                    event.threat_level_id = 0

            event.analysis = 0  # initial
            event.info = "ATD Analysis Report - {0}".format(self.mainfile)
            event.attributes = self.attributes
            event.Tag = 'ATD:Report'

            event = self.misp.add_event(event, pythonify=True)
            self.evenid = event.id
            print('SUCCESS: New MISP Event got created with ID: {}'.format(str(event.id)))

        except Exception as e:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            print("ERROR: Error in {location}.{funct_name}() - line {line_no} : {error}"
                  .format(location=__name__, funct_name=sys._getframe().f_code.co_name, line_no=exc_tb.tb_lineno,
                          error=str(e)))
Beispiel #48
0
        samp_details=json.loads(details_request.text)
        return samp_details.get('data')[0] if (samp_details.get('query_status') == 'ok') else None
    except:
        print(details_request.content)

    return None

# a dict of samples in the Event, keyed with <k>sha256 containing <v>MISP-Object Files
samples={}

# a cache of all the attributes in the event
attributes={}

#misp setup
pm = PyMISP(misp_url, misp_key, ssl=misp_check_cert)
misp_event=MISPEvent()
search=pm.search(controller='events', eventinfo=misp_event_name)

if ( len(search) == 1):
    misp_event.load(search[0])

    #load 'samples' dictionary from misp_event
    for obj in misp_event.get('Object'):
        if (obj.name == "file"):
            existing_hash = obj.get_attributes_by_relation("sha256")[0]['value']
            samples.update({existing_hash : obj})

    #reload 'attributes' dictionary from misp_event
    for attr in misp_event.get('Attribute'):
        attributes.update({attr.value : attr})
else:
Beispiel #49
0
urlVap = "https://tap-api-v2.proofpoint.com/v2/people/vap?window=30"  # Window can be 14, 30, and 90 Days

headers = {'Authorization': "Basic " + proofpoint_key}

responseVap = requests.request("GET", urlVap, headers=headers)

jsonDataVap = json.loads(responseVap.text)

for alert in jsonDataVap["users"]:
    orgc = MISPOrganisation()
    orgc.name = 'Proofpoint'
    orgc.id = '#{ORGC.ID}'  # organisation id
    orgc.uuid = '#{ORGC.UUID}'  # organisation uuid
    # initialize and set MISPEvent()
    event = MISPEvent()
    event.Orgc = orgc
    event.info = 'Very Attacked Person ' + jsonDataVap["interval"]
    event.distribution = 0  # Optional, defaults to MISP.default_event_distribution in MISP config
    event.threat_level_id = 2  # setting this to 0 breaks the integration
    event.analysis = 0  # Optional, defaults to 0 (initial analysis)

    totalVapUsers = event.add_attribute('counter',
                                        jsonDataVap["totalVapUsers"],
                                        comment="Total VAP Users")

    averageAttackIndex = event.add_attribute('counter',
                                             jsonDataVap["averageAttackIndex"],
                                             comment="Average Attack Count")

    vapAttackIndexThreshold = event.add_attribute(
Beispiel #50
0
class APIVoidParser():
    def __init__(self, attribute):
        self.misp_event = MISPEvent()
        self.attribute = MISPAttribute()
        self.attribute.from_dict(**attribute)
        self.misp_event.add_attribute(**self.attribute)
        self.url = 'https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&'

    def get_results(self):
        if hasattr(self, 'result'):
            return self.result
        event = json.loads(self.misp_event.to_json())
        results = {key: event[key] for key in ('Attribute', 'Object')}
        return {'results': results}

    def parse_domain(self, apikey):
        feature = 'dnslookup'
        if requests.get(f'{self.url.format(feature, apikey)}stats').json(
        )['credits_remained'] < 0.13:
            self.result = {
                'error':
                'You do not have enough APIVoid credits to proceed your request.'
            }
            return
        mapping = {
            'A': 'resolution-of',
            'MX': 'mail-server-of',
            'NS': 'server-name-of'
        }
        dnslookup = requests.get(
            f'{self.url.format(feature, apikey)}action=dns-any&host={self.attribute.value}'
        ).json()
        for item in dnslookup['data']['records']['items']:
            record_type = item['type']
            try:
                relationship = mapping[record_type]
            except KeyError:
                continue
            self._handle_dns_record(item, record_type, relationship)
        ssl = requests.get(
            f'{self.url.format("sslinfo", apikey)}host={self.attribute.value}'
        ).json()
        self._parse_ssl_certificate(ssl['data']['certificate'])

    def _handle_dns_record(self, item, record_type, relationship):
        dns_record = MISPObject('dns-record')
        dns_record.add_attribute('queried-domain',
                                 type='domain',
                                 value=item['host'])
        attribute_type, feature = ('ip-dst',
                                   'ip') if record_type == 'A' else ('domain',
                                                                     'target')
        dns_record.add_attribute(f'{record_type.lower()}-record',
                                 type=attribute_type,
                                 value=item[feature])
        dns_record.add_reference(self.attribute.uuid, relationship)
        self.misp_event.add_object(**dns_record)

    def _parse_ssl_certificate(self, certificate):
        x509 = MISPObject('x509')
        fingerprint = 'x509-fingerprint-sha1'
        x509.add_attribute(fingerprint,
                           type=fingerprint,
                           value=certificate['fingerprint'])
        x509_mapping = {
            'subject': {
                'name': ('text', 'subject')
            },
            'issuer': {
                'common_name': ('text', 'issuer')
            },
            'signature': {
                'serial': ('text', 'serial-number')
            },
            'validity': {
                'valid_from': ('datetime', 'validity-not-before'),
                'valid_to': ('datetime', 'validity-not-after')
            }
        }
        certificate = certificate['details']
        for feature, subfeatures in x509_mapping.items():
            for subfeature, mapping in subfeatures.items():
                attribute_type, relation = mapping
                x509.add_attribute(relation,
                                   type=attribute_type,
                                   value=certificate[feature][subfeature])
        x509.add_reference(self.attribute.uuid, 'seen-by')
        self.misp_event.add_object(**x509)
Beispiel #51
0
class TestMISPEvent(unittest.TestCase):

    def setUp(self):
        self.maxDiff = None
        self.mispevent = MISPEvent()

    def init_event(self):
        self.mispevent.info = 'This is a test'
        self.mispevent.distribution = 1
        self.mispevent.threat_level_id = 1
        self.mispevent.analysis = 1
        self.mispevent.set_date("2017-12-31")  # test the set date method

    def test_simple(self):
        with open('tests/mispevent_testfiles/simple.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_event(self):
        self.init_event()
        self.mispevent.publish()
        with open('tests/mispevent_testfiles/event.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_loadfile(self):
        self.mispevent.load_file('tests/mispevent_testfiles/event.json')
        with open('tests/mispevent_testfiles/event.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_event_tag(self):
        self.init_event()
        self.mispevent.add_tag('bar')
        self.mispevent.add_tag(name='baz')
        new_tag = MISPTag()
        new_tag.from_dict(name='foo')
        self.mispevent.add_tag(new_tag)
        with open('tests/mispevent_testfiles/event_tags.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_attribute(self):
        self.init_event()
        self.mispevent.add_attribute('filename', 'bar.exe')
        self.mispevent.add_attribute_tag('osint', 'bar.exe')
        attr_tags = self.mispevent.get_attribute_tag('bar.exe')
        self.assertEqual(self.mispevent.attributes[0].tags[0].name, 'osint')
        self.assertEqual(attr_tags[0].name, 'osint')
        with open('tests/mispevent_testfiles/attribute.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
        # Fake setting an attribute ID for testing
        self.mispevent.attributes[0].id = 42
        self.mispevent.delete_attribute(42)
        with open('tests/mispevent_testfiles/attribute_del.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_object_tag(self):
        self.mispevent.add_object(name='file', strict=True)
        self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}])
        self.assertEqual(self.mispevent.objects[0].attributes[0].tags[0].name, 'blah')
        self.assertTrue(self.mispevent.objects[0].has_attributes_by_relation(['filename']))
        self.assertEqual(len(self.mispevent.objects[0].get_attributes_by_relation('filename')), 1)
        self.mispevent.add_object(name='url', strict=True)
        self.mispevent.objects[1].add_attribute('url', value='https://www.circl.lu')
        self.mispevent.objects[0].uuid = 'a'
        self.mispevent.objects[1].uuid = 'b'
        self.mispevent.objects[0].add_reference('b', 'baz', comment='foo')
        self.assertEqual(self.mispevent.objects[0].references[0].relationship_type, 'baz')
        with open('tests/mispevent_testfiles/event_obj_attr_tag.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    @unittest.skip("Not supported on MISP: https://github.com/MISP/MISP/issues/2638 - https://github.com/MISP/PyMISP/issues/168")
    def test_object_level_tag(self):
        self.mispevent.add_object(name='file', strict=True)
        self.mispevent.objects[0].add_attribute('filename', value='bar')
        self.mispevent.objects[0].add_tag('osint')
        self.mispevent.objects[0].uuid = 'a'
        with open('tests/mispevent_testfiles/event_obj_tag.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_malware(self):
        with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
            pseudofile = BytesIO(f.read())
        self.init_event()
        self.mispevent.add_attribute('malware-sample', 'bar.exe', data=pseudofile)
        attribute = self.mispevent.attributes[0]
        self.assertEqual(attribute.malware_binary, pseudofile)
        with open('tests/mispevent_testfiles/malware.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_existing_malware(self):
        self.mispevent.load_file('tests/mispevent_testfiles/malware_exist.json')
        with open('tests/mispevent_testfiles/simple.json', 'rb') as f:
            pseudofile = BytesIO(f.read())
        self.assertEqual(
            self.mispevent.objects[0].get_attributes_by_relation('malware-sample')[0].malware_binary.read(),
            pseudofile.read())

    def test_sighting(self):
        sighting = MISPSighting()
        sighting.from_dict(value='1', type='bar', timestamp=11111111)
        with open('tests/mispevent_testfiles/sighting.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(sighting.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_existing_event(self):
        self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
        with open('tests/mispevent_testfiles/existing_event.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_shadow_attributes_existing(self):
        self.mispevent.load_file('tests/mispevent_testfiles/shadow.json')
        with open('tests/mispevent_testfiles/shadow.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_shadow_attributes(self):
        self.init_event()
        self.mispevent.add_proposal(type='filename', value='baz.jpg')
        self.mispevent.add_attribute('filename', 'bar.exe')
        self.mispevent.attributes[0].add_proposal(type='filename', value='bar.pdf')
        with open('tests/mispevent_testfiles/proposals.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_default_attributes(self):
        self.mispevent.add_object(name='file', strict=True)
        self.mispevent.objects[0].add_attribute('filename', value='bar', Tag=[{'name': 'blah'}])
        self.mispevent.add_object(name='file', strict=False, default_attributes_parameters=self.mispevent.objects[0].attributes[0])
        self.mispevent.objects[1].add_attribute('filename', value='baz')
        self.mispevent.objects[0].uuid = 'a'
        self.mispevent.objects[1].uuid = 'b'
        with open('tests/mispevent_testfiles/event_obj_def_param.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_obj_default_values(self):
        self.init_event()
        self.mispevent.add_object(name='whois', strict=True)
        self.mispevent.objects[0].add_attribute('registrar', value='registar.example.com')
        self.mispevent.objects[0].add_attribute('domain', value='domain.example.com')
        self.mispevent.objects[0].add_attribute('nameserver', value='ns1.example.com')
        self.mispevent.objects[0].add_attribute('nameserver', value='ns2.example.com', disable_correlation=False, to_ids=True, category='External analysis')
        self.mispevent.objects[0].uuid = 'a'
        with open('tests/mispevent_testfiles/def_param.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_event_not_edited(self):
        self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)

    def test_event_edited(self):
        self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
        self.mispevent.info = 'blah'
        self.assertTrue(self.mispevent.edited)

    def test_event_tag_edited(self):
        self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)
        self.mispevent.add_tag('foo')
        self.assertTrue(self.mispevent.edited)

    def test_event_attribute_edited(self):
        self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
        self.mispevent.attributes[0].value = 'blah'
        self.assertTrue(self.mispevent.attributes[0].edited)
        self.assertFalse(self.mispevent.attributes[1].edited)
        self.assertTrue(self.mispevent.edited)

    def test_event_attribute_tag_edited(self):
        self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)
        self.mispevent.attributes[0].tags[0].name = 'blah'
        self.assertTrue(self.mispevent.attributes[0].tags[0].edited)
        self.assertFalse(self.mispevent.attributes[0].tags[1].edited)
        self.assertTrue(self.mispevent.attributes[0].edited)
        self.assertTrue(self.mispevent.edited)

    def test_event_attribute_tag_edited_second(self):
        self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)
        self.mispevent.attributes[0].add_tag(name='blah')
        self.assertTrue(self.mispevent.attributes[0].edited)
        self.assertTrue(self.mispevent.edited)

    def test_event_object_edited(self):
        self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)
        self.mispevent.objects[0].comment = 'blah'
        self.assertTrue(self.mispevent.objects[0].edited)
        self.assertFalse(self.mispevent.objects[1].edited)
        self.assertTrue(self.mispevent.edited)

    def test_event_object_attribute_edited(self):
        self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)
        self.mispevent.objects[0].attributes[0].comment = 'blah'
        self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
        self.assertTrue(self.mispevent.objects[0].edited)
        self.assertTrue(self.mispevent.edited)

    def test_event_object_attribute_edited_tag(self):
        self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
        self.assertFalse(self.mispevent.edited)
        self.mispevent.objects[0].attributes[0].add_tag('blah')
        self.assertTrue(self.mispevent.objects[0].attributes[0].edited)
        self.assertTrue(self.mispevent.objects[0].edited)
        self.assertTrue(self.mispevent.edited)
        with open('tests/mispevent_testfiles/existing_event_edited.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))

    def test_obj_by_id(self):
        self.mispevent.load_file('tests/mispevent_testfiles/existing_event.json')
        misp_obj = self.mispevent.get_object_by_id(1556)
        self.assertEqual(misp_obj.uuid, '5a3cd604-e11c-4de5-bbbf-c170950d210f')

    def test_userdefined_object(self):
        self.init_event()
        self.mispevent.add_object(name='test_object_template', strict=True, misp_objects_path_custom='tests/mispevent_testfiles')
        with self.assertRaises(InvalidMISPObject) as e:
            # Fail on required
            self.mispevent.to_json()
        if sys.version_info >= (3, ):
            self.assertEqual(e.exception.message, '{\'member3\'} are required.')
        else:
            # Python2 bullshit
            self.assertEqual(e.exception.message, 'set([u\'member3\']) are required.')

        self.mispevent.objects[0].add_attribute('member3', value='foo')
        with self.assertRaises(InvalidMISPObject) as e:
            # Fail on requiredOneOf
            self.mispevent.to_json()
        self.assertEqual(e.exception.message, 'At least one of the following attributes is required: member1, member2')

        self.mispevent.objects[0].add_attribute('member1', value='bar')
        self.mispevent.objects[0].add_attribute('member1', value='baz')
        with self.assertRaises(InvalidMISPObject) as e:
            # member1 is not a multiple
            self.mispevent.to_json()
        self.assertEqual(e.exception.message, 'Multiple occurrences of member1 is not allowed')

        self.mispevent.objects[0].attributes = self.mispevent.objects[0].attributes[:2]
        self.mispevent.objects[0].uuid = 'a'
        with open('tests/mispevent_testfiles/misp_custom_obj.json', 'r') as f:
            ref_json = json.load(f)
        self.assertEqual(self.mispevent.to_json(), json.dumps(ref_json, sort_keys=True, indent=2))
Beispiel #52
0
 def __init__(self, attribute):
     self.misp_event = MISPEvent()
     self.attribute = MISPAttribute()
     self.attribute.from_dict(**attribute)
     self.misp_event.add_attribute(**self.attribute)
     self.url = 'https://endpoint.apivoid.com/{}/v1/pay-as-you-go/?key={}&'
Beispiel #53
0
class TestPDFExport(unittest.TestCase):

    def setUp(self):
        self.maxDiff = None
        self.mispevent = MISPEvent()
        if not manual_testing:
            self.root = "tests/"
        else:
            self.root = ""
        self.test_folder = self.root + "reportlab_testfiles/"
        self.test_batch_folder = self.root + "OSINT_output/"
        self.storage_folder_OSINT = self.root + "OSINT_PDF/"
        self.test_image_folder = self.root + "image_json/"
        self.storage_folder = self.root + "reportlab_testoutputs/"
        self.storage_image_folder = self.root + "reportlab_test_image_outputs/"
        self.moduleconfig = ["MISP_base_url_for_dynamic_link", "MISP_name_for_metadata", "Activate_textual_description",
                             "Activate_galaxy_description", "Activate_related_events", "Activate_internationalization_fonts", "Custom_fonts_path"]

    def init_event(self):
        self.mispevent.info = 'This is a test'
        self.mispevent.distribution = 1
        self.mispevent.threat_level_id = 1
        self.mispevent.analysis = 1
        self.mispevent.set_date("2017-12-31")  # test the set date method

    def check_python_2(self):
        if sys.version_info.major < 3:
            # we want Python2 test to pass
            return True

    def test_basic_event(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:
            self.init_event()
            reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
                                                       self.storage_folder + "basic_event.pdf")

    def test_event(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:
            self.init_event()
            self.mispevent.load_file(self.test_folder + 'to_delete1.json')
            reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
                                                       self.storage_folder + "normal_event.pdf")

    def test_HTML_json(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:
            self.init_event()
            self.mispevent.load_file(self.test_folder + 'HTML_event.json')
            reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
                                                       self.storage_folder + "HTML_event.pdf")

    def test_long_json(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:
            self.init_event()
            self.mispevent.load_file(self.test_folder + 'long_event.json')
            reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
                                                       self.storage_folder + "long_event.pdf")
            # Issue report : "We are not smart enough" : https://pairlist2.pair.net/pipermail/reportlab-users/2010-May/009529.html
            # Not nice but working solution exposed there: https://pairlist2.pair.net/pipermail/reportlab-users/2016-March/011525.html

    def test_very_long_json(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:
            self.init_event()
            self.mispevent.load_file(self.test_folder + 'very_long_event.json')
            reportlab_generator.register_value_to_file(reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
                                                       self.storage_folder + "very_long_event.pdf")

    def test_full_config_json(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:

            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'very_long_event.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "config_complete_event.pdf")

    def test_partial_0_config_json(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:

            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'very_long_event.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "config_partial_0_event.pdf")

    def test_partial_1_config_json(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:

            config = {}
            config[self.moduleconfig[1]] = "My Wonderful CERT"

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'very_long_event.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "config_partial_1_event.pdf")

    def test_image_json(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:

            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'image_event.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "image_event.pdf")

    def test_objects_1_json(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:

            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'mainly_objects_1.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "mainly_objects_1.pdf")

    def test_objects_2_json(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:

            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'mainly_objects_2.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "mainly_objects_2.pdf")

    def test_sightings_1_json(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:

            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'sighting_1.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "sighting_1.pdf")

    def test_sightings_2_json(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:

            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'sighting_2.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "sighting_2.pdf")

    def test_textual_json(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:

            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"
            config[self.moduleconfig[2]] = True

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'very_long_event.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "textual.pdf")

    def test_galaxy_1(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:
            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"
            config[self.moduleconfig[2]] = True
            config[self.moduleconfig[3]] = True

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'galaxy_1.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "galaxy_1.pdf")

    def test_related_events(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:
            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"
            config[self.moduleconfig[2]] = True
            config[self.moduleconfig[3]] = True
            config[self.moduleconfig[4]] = True

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'galaxy_1.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "related_events.pdf")

    def test_related_events_too_simple(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:
            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"
            config[self.moduleconfig[2]] = True
            config[self.moduleconfig[3]] = True
            config[self.moduleconfig[4]] = True

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'to_delete1.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "related_events_no_related.pdf")

    def test_utf(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:
            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"
            config[self.moduleconfig[2]] = True
            config[self.moduleconfig[3]] = True
            config[self.moduleconfig[4]] = True
            config[self.moduleconfig[5]] = True

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'japanese_test.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "japanese_test.pdf")

    def test_utf_heavy(self):
        if self.check_python_2():
            self.assertTrue(True)
        else:
            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"
            config[self.moduleconfig[2]] = True
            config[self.moduleconfig[3]] = True
            config[self.moduleconfig[4]] = True
            config[self.moduleconfig[5]] = True

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'japanese_test_heavy.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "japanese_test_heavy.pdf")

    def test_utf_ArialUNI_custompath(self):
        if self.check_python_2():
            self.assertTrue(True)
        elif not manual_testing:
            self.assertTrue(True)
        else:
            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"
            config[self.moduleconfig[2]] = True
            config[self.moduleconfig[3]] = True
            config[self.moduleconfig[4]] = True
            config[self.moduleconfig[5]] = True
            config[self.moduleconfig[6]] = "/home/user/Desktop/PyMISP/pymisp/tools/pdf_fonts/arial-unicode-ms/ARIALUNI.TTF"

            self.init_event()
            self.mispevent.load_file(self.test_folder + 'japanese_test_heavy.json')
            reportlab_generator.register_value_to_file(
                reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                self.storage_folder + "custom_path.pdf")

    def test_batch_image_events(self):
        # Test case ONLY for manual testing. Needs to download a full list of image events !

        if self.check_python_2():
            self.assertTrue(True)
        elif not manual_testing:
            self.assertTrue(True)
        else:
            self.init_event()

            file_nb = str(len(os.listdir(self.test_image_folder)))
            i = 0
            t = time.time()
            for curr_file in os.listdir(self.test_image_folder):
                self.mispevent = MISPEvent()
                file_path = self.test_image_folder + curr_file

                print("Current file : " + file_path + " " + str(i) + " over " + file_nb)
                i += 1

                self.mispevent.load_file(file_path)

                reportlab_generator.register_value_to_file(
                    reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
                    self.storage_image_folder + curr_file + ".pdf")
            print("Elapsed time : " + str(time.time() - t))
            # Local run : 73.061s for 102 files

    def test_batch_OSINT_events(self):
        # Test case ONLY for manual testing. Needs to download a full list of OSINT events !

        if self.check_python_2():
            self.assertTrue(True)
        elif not manual_testing:
            self.assertTrue(True)
        else:
            self.init_event()

            file_nb = str(len(os.listdir(self.test_batch_folder)))
            i = 0
            t = time.time()
            for curr_file in os.listdir(self.test_batch_folder):
                self.mispevent = MISPEvent()
                file_path = self.test_batch_folder + curr_file

                print("Current file : " + file_path + " " + str(i) + " over " + file_nb)
                i += 1

                self.mispevent.load_file(file_path)

                reportlab_generator.register_value_to_file(
                    reportlab_generator.convert_event_in_pdf_buffer(self.mispevent),
                    self.storage_folder_OSINT + curr_file + ".pdf")
            print("Elapsed time : " + str(time.time() - t))
            # Local run : 1958.930s for 1064 files

    def test_batch_OSINT_with_config_events(self):
        # Test case ONLY for manual testing. Needs to download a full list of OSINT events !

        if self.check_python_2():
            self.assertTrue(True)
        elif not manual_testing:
            self.assertTrue(True)
        else:
            self.init_event()

            config = {}
            config[self.moduleconfig[0]] = "http://localhost:8080"
            config[self.moduleconfig[1]] = "My Wonderful CERT"
            config[self.moduleconfig[2]] = True
            config[self.moduleconfig[3]] = True
            config[self.moduleconfig[4]] = True
            config[self.moduleconfig[5]] = True

            file_nb = str(len(os.listdir(self.test_batch_folder)))
            i = 0
            t = time.time()
            for curr_file in os.listdir(self.test_batch_folder):
                self.mispevent = MISPEvent()
                file_path = self.test_batch_folder + curr_file

                print("Current file : " + file_path + " " + str(i) + " over " + file_nb)
                i += 1

                self.mispevent.load_file(file_path)

                reportlab_generator.register_value_to_file(
                    reportlab_generator.convert_event_in_pdf_buffer(self.mispevent, config),
                    self.storage_folder_OSINT + curr_file + ".pdf")
            print("Elapsed time : " + str(time.time() - t))
Beispiel #54
0
    def __init__(self):
        super(MISP, self).__init__()
        self.cur_path = __project__.get_path()
        self.parser.add_argument("--url", help='URL of the MISP instance')
        self.parser.add_argument(
            "--off",
            action='store_true',
            help='Use offline (can only work on pre-downloaded events)')
        self.parser.add_argument("--on",
                                 action='store_true',
                                 help='Switch to online mode')
        self.parser.add_argument("-k",
                                 "--key",
                                 help='Your key on the MISP instance')
        self.parser.add_argument(
            "-v",
            "--verify",
            action='store_false',
            help='Disable certificate verification (for self-signed)')
        subparsers = self.parser.add_subparsers(dest='subname')

        # ##### Upload sample to MISP #####
        parser_up = subparsers.add_parser(
            'upload',
            help='Send malware sample to MISP.',
            formatter_class=argparse.RawDescriptionHelpFormatter,
            description=textwrap.dedent('''
                                            Distribution levels:
                                                * 0: Your organisation only
                                                * 1: This community only
                                                * 2: Connected communities
                                                * 3: All communities
                                                * 5: Inherit

                                            Sample categories:
                                                * 0: Payload delivery
                                                * 1: Artifacts dropped
                                                * 2: Payload installation
                                                * 3: External analysis

                                            Analysis levels:
                                                * 0: Initial
                                                * 1: Ongoing
                                                * 2: Completed

                                            Threat levels:
                                                * 0: High
                                                * 1: Medium
                                                * 2: Low
                                                * 3: Undefined
                                          '''))
        parser_up.add_argument(
            "-e",
            "--event",
            type=int,
            help=
            "Event ID to update. If None, and you're not connected to a MISP event a new one is created."
        )
        parser_up.add_argument(
            "-d",
            "--distrib",
            type=int,
            choices=[0, 1, 2, 3, 5],
            help="Distribution of the attributes for the new event.")
        parser_up.add_argument(
            "-s",
            "--sharing",
            type=int,
            help="Sharing group ID when distribution is set to 4.")
        parser_up.add_argument(
            "-ids",
            action='store_true',
            help="Is eligible for automatically creating IDS signatures.")
        parser_up.add_argument("-c",
                               "--categ",
                               type=int,
                               choices=[0, 1, 2, 3],
                               default=1,
                               help="Category of the samples.")
        parser_up.add_argument("-i",
                               "--info",
                               nargs='+',
                               help="Event info field of a new event.")
        parser_up.add_argument("-o",
                               "--comment",
                               nargs='+',
                               help="Comment associated to the sample.")
        parser_up.add_argument("-a",
                               "--analysis",
                               type=int,
                               choices=[0, 1, 2],
                               help="Analysis level a new event.")
        parser_up.add_argument("-t",
                               "--threat",
                               type=int,
                               choices=[0, 1, 2, 3],
                               help="Threat level of a new event.")

        # ##### Download samples from event #####
        parser_down = subparsers.add_parser(
            'download', help='Download malware samples from MISP.')
        group = parser_down.add_mutually_exclusive_group()
        group.add_argument(
            "-e",
            "--event",
            type=int,
            help="Download all the samples related to this event ID.")
        group.add_argument(
            "-l",
            "--list",
            nargs='*',
            help=
            "Download all the samples related to a list of events. Empty list to download all the samples of all the events stored in the current project."
        )  # noqa
        group.add_argument(
            "--hash",
            help="Download the sample related to this hash (only MD5).")

        # ##### Search in MISP #####
        parser_search = subparsers.add_parser(
            'search', help='Search in all the attributes.')
        parser_search.add_argument(
            "query",
            nargs='*',
            help=
            "String to search (if empty, search the hashes of the current file)."
        )

        # ##### Check hashes on VT #####
        parser_checkhashes = subparsers.add_parser(
            'check_hashes', help='Crosscheck hashes on VT.')
        parser_checkhashes.add_argument(
            "event",
            nargs='?',
            default=None,
            type=int,
            help="Lookup all the hashes of an event on VT.")
        parser_checkhashes.add_argument(
            "-p",
            "--populate",
            action='store_true',
            help="Automatically populate event with hashes found on VT.")

        # ##### Download Yara rules #####
        parser_checkhashes = subparsers.add_parser(
            'yara', help='Get YARA rules of an event.')
        parser_checkhashes.add_argument(
            "event",
            nargs='?',
            default=None,
            type=int,
            help="Download the yara rules of that event.")

        # ##### Get Events #####
        parser_pull = subparsers.add_parser(
            'pull', help='Initialize the session with an existing MISP event.')
        parser_pull.add_argument("event",
                                 nargs='+',
                                 type=int,
                                 help="(List of) Event(s) ID.")

        # ##### Create an Event #####
        parser_create_event = subparsers.add_parser(
            'create_event',
            help=
            'Create a new event on MISP and initialize the session with it.',
            formatter_class=argparse.RawDescriptionHelpFormatter,
            description=textwrap.dedent('''
                                                      Distribution levels:
                                                          * 0: Your organisation only
                                                          * 1: This community only
                                                          * 2: Connected communities
                                                          * 3: All communities
                                                          * 4: Sharing group

                                                      Sharing Group:
                                                          * #: ID of sharing group

                                                      Analysis levels:
                                                          * 0: Initial
                                                          * 1: Ongoing
                                                          * 2: Completed

                                                      Threat levels:
                                                          * 0: High
                                                          * 1: Medium
                                                          * 2: Low
                                                          * 3: Undefined
                                                    '''))
        parser_create_event.add_argument(
            "-d",
            "--distrib",
            type=int,
            choices=[0, 1, 2, 3, 4],
            help="Distribution of the attributes for the new event.")
        parser_create_event.add_argument(
            "-s",
            "--sharing",
            type=int,
            help="Sharing group ID when distribution is set to 4.")
        parser_create_event.add_argument("-t",
                                         "--threat",
                                         type=int,
                                         choices=[0, 1, 2, 3],
                                         help="Threat level of a new event.")
        parser_create_event.add_argument("-a",
                                         "--analysis",
                                         type=int,
                                         choices=[0, 1, 2],
                                         help="Analysis level a new event.")
        parser_create_event.add_argument(
            "-i",
            "--info",
            required=True,
            nargs='+',
            help="Event info field of a new event.")
        parser_create_event.add_argument(
            "--date", help="Date of the event. (Default: today).")

        # ##### Add Hashes #####
        h = subparsers.add_parser(
            "add_hashes",
            help="If no parameters, add all the hashes of the current session."
        )
        h.add_argument("-f", "--filename", help="Filename")
        h.add_argument("-m", "--md5", help="MD5")
        h.add_argument("-s", "--sha1", help="SHA1")
        h.add_argument("-a", "--sha256", help="SHA256")

        # ##### Add attributes #####
        parser_add = subparsers.add_parser(
            'add', help='Add attributes to an existing MISP event.')
        subparsers_add = parser_add.add_subparsers(dest='add')
        # Hashes
        # Generic add
        temp_me = MISPEvent()
        for t in sorted(temp_me.types):
            sp = subparsers_add.add_parser(
                t, help="Add {} to the event.".format(t))
            sp.add_argument(t, nargs='+')

        # ##### Show attributes  #####
        subparsers.add_parser(
            'show', help='Show attributes to an existing MISP event.')

        # ##### Open file #####
        o = subparsers.add_parser(
            'open', help='Open a sample from the temp directory.')
        ox = o.add_mutually_exclusive_group(required=True)
        ox.add_argument("-l",
                        "--list",
                        action='store_true',
                        help="List available files")
        ox.add_argument(
            "-d",
            "--delete",
            help=
            "Delete temporary files (use 'all' to remove all the local samples or an Event ID to only remove the associated samples)"
        )
        ox.add_argument("sid",
                        nargs='?',
                        type=int,
                        help='Sample ID to open (from the list option).')

        # ##### Publish an event #####
        subparsers.add_parser('publish',
                              help='Publish an existing MISP event.')

        # ##### Show version #####
        subparsers.add_parser('version',
                              help='Returns the version of the MISP instance.')

        # Store
        s = subparsers.add_parser(
            'store',
            help='Store the current MISP event in the current project.')
        s.add_argument("-l",
                       "--list",
                       action='store_true',
                       help="List stored MISP events")
        s.add_argument("-u",
                       "--update",
                       action='store_true',
                       help="Update all stored MISP events")
        s.add_argument(
            "-s",
            "--sync",
            action='store_true',
            help="Sync all MISP Events with the remote MISP instance")
        s.add_argument("-d",
                       "--delete",
                       type=int,
                       help="Delete a stored MISP event")
        s.add_argument("-o", "--open", help="Open a stored MISP event")

        # Tags
        s = subparsers.add_parser('tag',
                                  help='Tag managment using MISP taxonomies.')
        s.add_argument("-l",
                       "--list",
                       action='store_true',
                       help="List Existing taxonomies.")
        s.add_argument("-d",
                       "--details",
                       help="Display all values of a taxonomy.")
        s.add_argument("-s",
                       "--search",
                       help="Search all tags matching a value.")
        s.add_argument("-e", "--event", help="Add tag to the current event.")
        s.add_argument(
            "-a",
            "--attribute",
            nargs='+',
            help=
            "Add tag to an attribute of the current event. Syntax: <identifier for the attribute> <machinetag>"
        )

        # Admin
        s = subparsers.add_parser('admin', help='Administration options.')
        admin_parser = s.add_subparsers(dest='admin')
        # Organisation
        org = admin_parser.add_parser('org', help="Organisation managment.")
        subparsers_org = org.add_subparsers(dest='org')
        # Get
        display = subparsers_org.add_parser('display',
                                            help="Display an organisation.")
        display.add_argument(
            'id',
            help=
            'ID of the organisation to display. Use "local" to display all local organisations, "external" for all remote organisations, and "all", for both.'
        )
        # Search
        search = subparsers_org.add_parser(
            'search', help="Search an organisation by name.")
        search.add_argument('name', help='(Partial) name of the organisation.')
        search.add_argument(
            '-t',
            '--type',
            default='local',
            choices=['local', 'external', 'all'],
            help=
            'Use "local" to search in all local organisations, "external" for remote organisations, and "all", for both.'
        )
        # Add
        add_org = subparsers_org.add_parser('add', help="Add an organisation.")
        add_org.add_argument('name', help='Organisation name.')
        add_org.add_argument('-u',
                             '--uuid',
                             default=None,
                             help='UUID of the organisation.')
        add_org.add_argument('-d',
                             '--description',
                             default=[],
                             nargs='+',
                             help='Description of the organisation.')
        add_org.add_argument('-t',
                             '--type',
                             default=[],
                             nargs='+',
                             help='Type of the organisation.')
        add_org.add_argument('-n',
                             '--nationality',
                             default=None,
                             help='Nationality of the organisation.')
        add_org.add_argument('-s',
                             '--sector',
                             default=[],
                             nargs='+',
                             help='Sector of the organisation.')
        add_org.add_argument('-c',
                             '--contacts',
                             default=[],
                             nargs='+',
                             help='Contact point(s) in the organisation.')
        add_org.add_argument('--not-local',
                             default=True,
                             action='store_false',
                             help='**Not** a local organisation.')
        # Delete
        delete = subparsers_org.add_parser('delete',
                                           help="Delete an organisation.")
        delete.add_argument('id', help='ID of the organisation to delete.')
        # Edit
        edit = subparsers_org.add_parser('edit', help="Edit an organisation.")
        edit.add_argument('id', help='ID of the organisation to edit.')
        edit.add_argument('-n', '--name', help='Organisation name.')
        edit.add_argument('-u', '--uuid', help='UUID of the organisation.')
        edit.add_argument('-d',
                          '--description',
                          default=[],
                          nargs='+',
                          help='Description of the organisation.')
        edit.add_argument('-t',
                          '--type',
                          default=[],
                          nargs='+',
                          help='Type of the organisation.')
        edit.add_argument('--nationality',
                          help='Nationality of the organisation.')
        edit.add_argument('-s',
                          '--sector',
                          default=[],
                          nargs='+',
                          help='Sector of the organisation.')
        edit.add_argument('-c',
                          '--contacts',
                          default=[],
                          nargs='+',
                          help='Contact point(s) in the organisation.')
        edit.add_argument('--not-local',
                          default=True,
                          action='store_false',
                          help='**Not** a local organisation.')

        # User
        user = admin_parser.add_parser('user', help="User managment.")
        subparsers_user = user.add_subparsers(dest='user')
        # Get
        display = subparsers_user.add_parser('display', help="Display a user.")
        display.add_argument(
            'id',
            help='ID of the user to display. Use "all" to display all users.')
        # Search
        search = subparsers_user.add_parser('search',
                                            help="Search a user by email.")
        search.add_argument('name', help='(Partial) email of the user.')
        # Add
        add_usr = subparsers_user.add_parser('add', help="Add a user.")
        add_usr.add_argument('email', help='User email address.')
        add_usr.add_argument('-o',
                             '--org-id',
                             default=None,
                             help='Organisation ID of the user.')
        add_usr.add_argument('-r',
                             '--role-id',
                             default=None,
                             help='Role of the user')
        add_usr.add_argument('-g',
                             '--gpgkey',
                             default=None,
                             help='Path to the GPG public key export')
        add_usr.add_argument(
            '-c',
            '--change-pw',
            default=None,
            action='store_true',
            help='Force thanging the password after next login')
        add_usr.add_argument('-t',
                             '--termsaccepted',
                             default=None,
                             action='store_true',
                             help='Set the TOC to accepted')
        add_usr.add_argument('-p',
                             '--password',
                             default=None,
                             help='Set a new password')
        add_usr.add_argument('-d',
                             '--disabled',
                             default=None,
                             action='store_true',
                             help='Disable the account')
        # Delete
        delete = subparsers_user.add_parser('delete', help="Delete a user.")
        delete.add_argument('id', help='ID of the user to delete.')
        # Edit
        edit = subparsers_user.add_parser('edit', help="Edit a user.")
        edit.add_argument('id', help='ID of the user to edit.')
        edit.add_argument('-e', '--email', help='User email address.')
        edit.add_argument('-o',
                          '--org-id',
                          default=None,
                          help='Organisation ID of the user.')
        edit.add_argument('-r',
                          '--role-id',
                          default=None,
                          help='Role of the user')
        edit.add_argument('-g',
                          '--gpgkey',
                          default=None,
                          help='Path to the GPG public key export')
        edit.add_argument('-c',
                          '--change-pw',
                          default=None,
                          action='store_true',
                          help='Force thanging the password after next login')
        edit.add_argument('-t',
                          '--termsaccepted',
                          default=None,
                          action='store_true',
                          help='Set the TOC to accepted')
        edit.add_argument('-p',
                          '--password',
                          default=None,
                          help='Set a new password')
        edit.add_argument('-d',
                          '--disabled',
                          default=None,
                          action='store_true',
                          help='Disable the account')

        # Role
        role = admin_parser.add_parser('role', help="Role managment.")
        subparsers_role = role.add_subparsers(dest='role')
        # Get
        display = subparsers_role.add_parser('display',
                                             help="Display all the roles.")
        # Search
        search = subparsers_role.add_parser('search',
                                            help="Search a role by name.")
        search.add_argument('name', help='(Partial) name of the role.')

        # Tags
        t = admin_parser.add_parser('tag', help="Tag managment.")
        subparsers_tag = t.add_subparsers(dest='tag')
        # Get
        display = subparsers_tag.add_parser('display',
                                            help="Display all the tags.")
        # Search
        search = subparsers_tag.add_parser('search',
                                           help="Search a tag by name.")
        search.add_argument('name', help='(Partial) name of the tag.')

        self.categories = {
            0: 'Payload delivery',
            1: 'Artifacts dropped',
            2: 'Payload installation',
            3: 'External analysis'
        }
Beispiel #55
0
WHERE rel_cnt > 5
MATCH (m)-[r:has]->(n)
RETURN m, n LIMIT 200;
"""

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Get all the events matching a value.')
    parser.add_argument("-s", "--search", required=True, help="String to search.")
    parser.add_argument("--host", default='localhost:7474', help="Host where neo4j is running.")
    parser.add_argument("-u", "--user", default='neo4j', help="User on neo4j.")
    parser.add_argument("-p", "--password", default='neo4j', help="Password on neo4j.")
    parser.add_argument("-d", "--deleteall", action="store_true", default=False, help="Delete all nodes from the database")
    args = parser.parse_args()

    neo4j = Neo4j(args.host, args.user, args.password)
    if args.deleteall:
        neo4j.del_all()
    misp = PyMISP(misp_url, misp_key)
    result = misp.search_all(args.search)
    for json_event in result['response']:
        if not json_event['Event']:
            print(json_event)
            continue
        print('Importing', json_event['Event']['info'], json_event['Event']['id'])
        try:
            misp_event = MISPEvent()
            misp_event.load(json_event)
            neo4j.import_event(misp_event)
        except:
            print('broken')
Beispiel #56
0
class StixParser():
    def __init__(self):
        self.misp_event = MISPEvent()
        self.event = defaultdict(dict)
        self.misp_event['Galaxy'] = []

    def loadEvent(self, args):
        filename = os.path.join(os.path.dirname(args[0]), args[1])
        with open(filename, 'r', encoding='utf-8') as f:
            event = json.loads(f.read())
        self.filename = filename
        self.stix_version = 'stix {}'.format(event.get('spec_version'))
        for o in event.get('objects'):
            parsed_object = stix2.parse(o, allow_custom=True)
            try:
                object_type = parsed_object._type
            except AttributeError:
                object_type = parsed_object['type']
            object_uuid = parsed_object['id'].split('--')[1]
            if object_type.startswith('x-misp-object'):
                object_type = 'x-misp-object'
            self.event[object_type][object_uuid] = parsed_object
        if not self.event:
            print(
                json.dumps({
                    'success': 0,
                    'message': 'There is no valid STIX object to import'
                }))
            sys.exit(1)
        try:
            event_distribution = args[2]
            if not isinstance(event_distribution, int):
                event_distribution = int(
                    event_distribution) if event_distribution.isdigit() else 5
        except IndexError:
            event_distribution = 5
        try:
            attribute_distribution = args[3]
            if attribute_distribution != 'event' and not isinstance(
                    attribute_distribution, int):
                attribute_distribution = int(
                    attribute_distribution) if attribute_distribution.isdigit(
                    ) else 5
        except IndexError:
            attribute_distribution = 5
        self.misp_event.distribution = event_distribution
        self.__attribute_distribution = event_distribution if attribute_distribution == 'event' else attribute_distribution
        self.load_mapping()

    def load_mapping(self):
        self.objects_mapping = {
            'asn': {
                'observable': observable_asn,
                'pattern': pattern_asn
            },
            'domain-ip': {
                'observable': observable_domain_ip,
                'pattern': pattern_domain_ip
            },
            'email': {
                'observable': self.observable_email,
                'pattern': self.pattern_email
            },
            'file': {
                'observable': observable_file,
                'pattern': self.pattern_file
            },
            'ip-port': {
                'observable': observable_ip_port,
                'pattern': pattern_ip_port
            },
            'network-socket': {
                'observable': observable_socket,
                'pattern': pattern_socket
            },
            'process': {
                'observable': observable_process,
                'pattern': pattern_process
            },
            'registry-key': {
                'observable': observable_regkey,
                'pattern': pattern_regkey
            },
            'url': {
                'observable': observable_url,
                'pattern': pattern_url
            },
            'WindowsPEBinaryFile': {
                'observable': self.observable_pe,
                'pattern': self.pattern_pe
            },
            'x509': {
                'observable': observable_x509,
                'pattern': pattern_x509
            }
        }
        self.object_from_refs = {
            'course-of-action': self.parse_course_of_action,
            'vulnerability': self.parse_vulnerability,
            'x-misp-object': self.parse_custom
        }
        self.object_from_refs.update(
            dict.fromkeys(list(galaxy_types.keys()), self.parse_galaxy))
        self.object_from_refs.update(
            dict.fromkeys(['indicator', 'observed-data'],
                          self.parse_usual_object))

    def handler(self):
        self.outputname = '{}.stix2'.format(self.filename)
        if self.from_misp():
            self.buildMispDict()
        else:
            self.version_attribute = {
                'type': 'text',
                'object_relation': 'version',
                'value': self.stix_version
            }
            self.buildExternalDict()
        self.set_distribution()

    def from_misp(self):
        for _, o in self.event['report'].items():
            if o._type == 'report' and 'misp:tool="misp2stix2"' in o.get(
                    'labels'):
                return True
        return False

    def buildMispDict(self):
        report_attributes = defaultdict(list)
        orgs = []
        for _, report in self.event['report'].items():
            org_uuid = report['created_by_ref'].split('--')[1]
            if org_uuid not in orgs:
                orgs.append(org_uuid)
            report_name = report['name']
            if report_name not in orgs:
                report_attributes['name'].append(report_name)
            if report.get('published'):
                report_attributes['published'].append(report['published'])
            if hasattr(report, 'labels'):
                for l in report['labels']:
                    if l not in report_attributes['labels']:
                        report_attributes['labels'].append(l)
            if hasattr(report, 'external_references'):
                for e in report['external_references']:
                    self.add_link(e)
            for ref in report['object_refs']:
                object_type, uuid = ref.split('--')
                if object_type == 'relationship':
                    continue
                object2parse = self.event[object_type][uuid]
                labels = object2parse.get('labels')
                self.object_from_refs[object_type](object2parse, labels)
        if len(orgs) == 1:
            identity = self.event['identity'][orgs[0]]
            self.misp_event['Org'] = {'name': identity['name']}
        if len(report_attributes['published']) == 1:
            self.misp_event.publish_timestamp = self.getTimestampfromDate(
                report_attributes['published'][0])
        if len(report_attributes['name']) == 1:
            self.misp_event.info = report_attributes['name'][0]
        else:
            self.misp_event.info = "Imported from MISP import for STIX 2.0 script."
        for l in report_attributes['labels']:
            self.misp_event.add_tag(l)

    def add_link(self, e):
        link = {"type": "link"}
        comment = e.get('source_name')
        try:
            comment = comment.split('url - ')[1]
        except IndexError:
            pass
        if comment:
            link['comment'] = comment
        link['value'] = e.get('url')
        self.misp_event.add_attribute(**link)

    def parse_usual_object(self, o, labels):
        if 'from_object' in labels:
            self.parse_object(o, labels)
        else:
            self.parse_attribute(o, labels)

    def parse_galaxy(self, o, labels):
        galaxy_type = self.get_misp_type(labels)
        tag = labels[1]
        value = tag.split(':')[1].split('=')[1]
        galaxy_description, cluster_description = o.get('description').split(
            '|')
        _, uuid = o.get('id').split('--')
        galaxy = {
            'type':
            galaxy_type,
            'name':
            o.get('name'),
            'description':
            galaxy_description,
            'GalaxyCluster': [{
                'type': galaxy_type,
                'value': value,
                'tag_name': tag,
                'description': cluster_description,
                'uuid': uuid
            }]
        }
        self.misp_event['Galaxy'].append(galaxy)

    def parse_course_of_action(self, o, _):
        misp_object = MISPObject('course-of-action')
        if 'name' in o:
            attribute = {
                'type': 'text',
                'object_relation': 'name',
                'value': o.get('name')
            }
            misp_object.add_attribute(**attribute)
        else:
            return
        if 'description' in o:
            attribute = {
                'type': 'text',
                'object_relation': 'description',
                'value': o.get('description')
            }
            misp_object.add_attribute(**attribute)
        self.misp_event.add_object(**misp_object)

    def parse_custom(self, o, labels):
        if 'from_object' in labels:
            self.parse_custom_object(o)
        else:
            self.parse_custom_attribute(o, labels)

    def parse_custom_object(self, o):
        name = o.get('type').split('x-misp-object-')[1]
        timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp'))
        category = o.get('category')
        attributes = []
        values = o.get('x_misp_values')
        for v in values:
            attribute_type, object_relation = v.split('_')
            attribute = {
                'type': attribute_type,
                'value': values.get(v),
                'object_relation': object_relation
            }
            attributes.append(attribute)
        misp_object = {
            'name': name,
            'timestamp': timestamp,
            'meta-category': category,
            'Attribute': attributes
        }
        self.misp_event.add_object(**misp_object)

    def parse_custom_attribute(self, o, labels):
        attribute_type = o.get('type').split('x-misp-object-')[1]
        if attribute_type not in misp_types:
            attribute_type = attribute_type.replace('-', '|')
        timestamp = self.getTimestampfromDate(o.get('x_misp_timestamp'))
        to_ids = bool(labels[1].split('=')[1])
        value = o.get('x_misp_value')
        category = self.get_misp_category(labels)
        attribute = {
            'type': attribute_type,
            'timestamp': timestamp,
            'to_ids': to_ids,
            'value': value,
            'category': category
        }
        self.misp_event.add_attribute(**attribute)

    def parse_object(self, o, labels):
        object_type = self.get_misp_type(labels)
        name = 'file' if object_type == 'WindowsPEBinaryFile' else object_type
        object_category = self.get_misp_category(labels)
        stix_type = o._type
        misp_object = MISPObject(name)
        misp_object['meta-category'] = object_category
        if stix_type == 'indicator':
            pattern = o.get('pattern').replace('\\\\', '\\').split(' AND ')
            pattern[0] = pattern[0][1:]
            pattern[-1] = pattern[-1][:-1]
            attributes = self.objects_mapping[object_type]['pattern'](pattern)
        if stix_type == 'observed-data':
            observable = o.get('objects')
            attributes = self.objects_mapping[object_type]['observable'](
                observable)
        if isinstance(attributes, tuple):
            attributes, pe_uuid = attributes
            misp_object.add_reference(pe_uuid, 'included-in')
        for attribute in attributes:
            misp_object.add_attribute(**attribute)
        misp_object.to_ids = bool(labels[1].split('=')[1])
        self.misp_event.add_object(**misp_object)

    def parse_attribute(self, o, labels):
        attribute_type = self.get_misp_type(labels)
        attribute_category = self.get_misp_category(labels)
        attribute = {'type': attribute_type, 'category': attribute_category}
        tags = [{'name': label} for label in labels[3:]]
        if tags:
            attribute['Tag'] = tags
        stix_type = o._type
        if stix_type == 'vulnerability':
            value = o.get('name')
        else:
            if stix_type == 'indicator':
                o_date = o.get('valid_from')
                pattern = o.get('pattern').replace('\\\\', '\\')
                value = self.parse_pattern_with_data(
                    pattern) if attribute_type in (
                        'malware-sample',
                        'attachment') else self.parse_pattern(pattern)
                attribute['to_ids'] = True
            else:
                o_date = o.get('first_observed')
                observable = o.get('objects')
                try:
                    value = self.parse_observable(observable, attribute_type)
                except:
                    print('{}: {}'.format(attribute_type, observable))
                attribute['to_ids'] = False
            attribute['timestamp'] = self.getTimestampfromDate(o_date)
        if 'description' in o:
            attribute['comment'] = o.get('description')
        if isinstance(value, tuple):
            value, data = value
            attribute['data'] = io.BytesIO(data.encode())
        attribute['value'] = value
        self.misp_event.add_attribute(**attribute)

    def parse_vulnerability(self, o, labels):
        if len(labels) > 2:
            self.parse_usual_object(o, labels)
        else:
            self.parse_galaxy(o, labels)

    @staticmethod
    def observable_email(observable):
        attributes = []
        addresses = {}
        files = {}
        for o_key, o_dict in observable.items():
            part_type = o_dict._type
            if part_type == 'email-addr':
                addresses[o_key] = o_dict.get('value')
            elif part_type == 'file':
                files[o_key] = o_dict.get('name')
            else:
                message = dict(o_dict)
        attributes.append({
            'type': 'email-src',
            'object_relation': 'from',
            'value': addresses[message.pop('from_ref')],
            'to_ids': False
        })
        for ref in ('to_refs', 'cc_refs'):
            if ref in message:
                for item in message.pop(ref):
                    mapping = email_mapping[ref]
                    attributes.append({
                        'type': mapping['type'],
                        'object_relation': mapping['relation'],
                        'value': addresses[item],
                        'to_ids': False
                    })
        if 'body_multipart' in message:
            for f in message.pop('body_multipart'):
                attributes.append({
                    'type': 'email-attachment',
                    'object_relation': 'attachment',
                    'value': files[f.get('body_raw_ref')],
                    'to_ids': False
                })
        for m_key, m_value in message.items():
            if m_key == 'additional_header_fields':
                for field_key, field_value in m_value.items():
                    mapping = email_mapping[field_key]
                    if field_key == 'Reply-To':
                        for rt in field_value:
                            attributes.append({
                                'type':
                                mapping['type'],
                                'object_relation':
                                mapping['relation'],
                                'value':
                                rt,
                                'to_ids':
                                False
                            })
                    else:
                        attributes.append({
                            'type':
                            mapping['type'],
                            'object_relation':
                            mapping['relation'],
                            'value':
                            field_value,
                            'to_ids':
                            False
                        })
            else:
                try:
                    mapping = email_mapping[m_key]
                    attributes.append({
                        'type': mapping['type'],
                        'object_relation': mapping['relation'],
                        'value': m_value,
                        'to_ids': False
                    })
                except KeyError:
                    if m_key.startswith("x_misp_attachment_"):
                        attribute_type, relation = m_key.split(
                            "x_misp_")[1].split("_")
                        attributes.append({
                            'type':
                            attribute_type,
                            'object_relation':
                            relation,
                            'to_ids':
                            False,
                            'value':
                            m_value['value'],
                            'data':
                            io.BytesIO(m_value['data'].encode())
                        })
                    elif "x_misp_" in m_key:
                        attribute_type, relation = m_key.split(
                            "x_misp_")[1].split("_")
                        attributes.append({
                            'type': attribute_type,
                            'object_relation': relation,
                            'value': m_value,
                            'to_ids': False
                        })
        return attributes

    @staticmethod
    def pattern_email(pattern):
        attributes = []
        attachments = defaultdict(dict)
        for p in pattern:
            p_type, p_value = p.split(' = ')
            try:
                mapping = email_mapping[p_type]
                attributes.append({
                    'type': mapping['type'],
                    'object_relation': mapping['relation'],
                    'value': p_value[1:-1],
                    'to_ids': True
                })
            except KeyError:
                if p_type.startswith("email-message:'x_misp_attachment_"):
                    relation, field = p_type.split('.')
                    relation = relation.split(':')[1][1:-1]
                    attachments[relation][field] = p_value[1:-1]
                elif "x_misp_" in p_type:
                    attribute_type, relation = p_type.split(
                        "x_misp_")[1][:-1].split("_")
                    attributes.append({
                        'type': attribute_type,
                        'object_relation': relation,
                        'value': p_value[1:-1],
                        'to_ids': True
                    })
        for a_key, a_dict in attachments.items():
            _, _, attribute_type, relation = a_key.split('_')
            attributes.append({
                'type': attribute_type,
                'object_relation': relation,
                'to_ids': True,
                'value': a_dict['value'],
                'data': io.BytesIO(a_dict['data'].encode())
            })
        return attributes

    @staticmethod
    def pattern_file(pattern):
        attributes = []
        malware_sample = {}
        for p in pattern:
            p_type, p_value = p.split(' = ')
            if p_type == 'artifact:payload_bin':
                malware_sample['data'] = p_value
            elif p_type in ("file:name", "file:hashes.'md5'"):
                try:
                    mapping = file_mapping[p_type]
                    attributes.append({
                        'type': mapping['type'],
                        'object_relation': mapping['relation'],
                        'value': p_value[1:-1],
                        'to_ids': True
                    })
                    malware_sample['filename'] = p_value[1:-1]
                except KeyError:
                    attributes.append({
                        'type': 'md5',
                        'object_relation': 'md5',
                        'value': p_value[1:-1],
                        'to_ids': True
                    })
                    malware_sample['md5'] = p_value[1:-1]
            elif 'file:hashes.' in p_type:
                _, h = p_type.split('.')
                h = h[1:-1]
                attributes.append({
                    'type': h,
                    'object_relation': h,
                    'value': p_value[1:-1]
                })
            else:
                try:
                    mapping = file_mapping[p_type]
                    attributes.append({
                        'type': mapping['type'],
                        'object_relation': mapping['relation'],
                        'value': p_value[1:-1],
                        'to_ids': True
                    })
                except KeyError:
                    if "x_misp_" in p_type:
                        attribute_type, relation = p_type.split(
                            "x_misp_")[1][:-1].split("_")
                        attributes.append({
                            'type': attribute_type,
                            'object_relation': relation,
                            'value': p_value[1:-1],
                            'to_ids': True
                        })
        if 'data' in malware_sample:
            value = "{}|{}".format(malware_sample['filename'],
                                   malware_sample['md5'])
            attributes.append({
                'type':
                'malware-sample',
                'object_relation':
                'malware-sample',
                'value':
                value,
                'to_ids':
                True,
                'data':
                io.BytesIO(malware_sample['data'].encode())
            })
        return attributes

    def observable_pe(self, observable):
        extension = observable['1']['extensions']['windows-pebinary-ext']
        sections = extension['sections']
        pe = MISPObject('pe')
        pe_uuid = str(uuid.uuid4())
        pe.uuid = pe_uuid
        self.fill_object_attributes_observable(pe, pe_mapping, extension)
        for section in sections:
            pe_section = MISPObject('pe-section')
            if 'hashes' in section:
                for h_type, h_value in section['hashes'].items():
                    h_type = h_type.lower().replace('-', '')
                    pe_section.add_attribute(
                        **{
                            'type': h_type,
                            'object_relation': h_type,
                            'value': h_value,
                            'to_ids': False
                        })
            self.fill_object_attributes_observable(pe_section,
                                                   pe_section_mapping, section)
            section_uuid = str(uuid.uuid4())
            pe_section.uuid = section_uuid
            pe.add_reference(section_uuid, 'included-in')
            self.misp_event.add_object(**pe_section)
        self.misp_event.add_object(**pe)
        return observable_file(observable), pe_uuid

    @staticmethod
    def fill_object_attributes_observable(misp_object, mapping_dict,
                                          stix_object):
        for stix_type, value in stix_object.items():
            try:
                mapping = mapping_dict[stix_type]
                misp_object.add_attribute(
                    **{
                        'type': mapping['type'],
                        'object_relation': mapping['relation'],
                        'value': value,
                        'to_ids': False
                    })
            except KeyError:
                if stix_type.startswith("x_misp_"):
                    attribute_type, relation = parse_custom_property(stix_type)
                    misp_object.add_attribute(
                        **{
                            'type': attribute_type,
                            'object_relation': relation[:-1],
                            'value': value,
                            'to_ids': False
                        })

    def pattern_pe(self, pattern):
        attributes = []
        sections = defaultdict(dict)
        pe = MISPObject('pe')
        pe_uuid = str(uuid.uuid4())
        pe.uuid = pe_uuid
        for p in pattern:
            p_type, p_value = p.split(' = ')
            p_value = p_value[1:-1]
            if ':extensions.' in p_type:
                if '.sections[' in p_type:
                    p_type_list = p_type.split('.')
                    stix_type = "hashes.{}".format(
                        p_type_list[4]
                        [1:-1]) if '.hashes.' in p_type else p_type_list[3]
                    sections[p_type_list[2]][stix_type] = p_value
                else:
                    stix_type = p_type.split('.')[-1]
                    try:
                        mapping = pe_mapping[stix_type]
                        pe.add_attribute(
                            **{
                                'type': mapping['type'],
                                'object_relation': mapping['relation'],
                                'value': p_value,
                                'to_ids': True
                            })
                    except KeyError:
                        if stix_type.startswith("x_misp_"):
                            attribute_type, relation = parse_custom_property(
                                stix_type)
                            pe.add_attribute(
                                **{
                                    'type': attribute_type,
                                    'object_relation': relation[:-2],
                                    'value': p_value,
                                    'to_ids': False
                                })
            else:
                if 'file:hashes.' in p_type:
                    _, h = p_type.split('.')
                    h = h[1:-1]
                    attributes.append({
                        'type': h,
                        'object_relation': h,
                        'value': p_value,
                        'to_ids': True
                    })
                else:
                    try:
                        mapping = file_mapping[p_type]
                        attributes.append({
                            'type':
                            mapping['type'],
                            'object_relation':
                            mapping['relation'],
                            'value':
                            p_value,
                            'to_ids':
                            True
                        })
                    except KeyError:
                        if "x_misp_" in p_type:
                            attribute_type, relation = p_type.split(
                                "x_misp_")[1][:-1].split("_")
                            attributes.append({
                                'type': attribute_type,
                                'object_relation': relation,
                                'value': p_value,
                                'to_ids': True
                            })
        for _, section in sections.items():
            pe_section = MISPObject('pe-section')
            for stix_type, value in section.items():
                if 'hashes.' in stix_type:
                    h_type = stix_type.split('.')[1]
                    pe_section.add_attribute(
                        **{
                            'type': h_type,
                            'object_relation': h_type,
                            'value': value,
                            'to_ids': True
                        })
                else:
                    try:
                        mapping = pe_section_mapping[stix_type]
                        pe_section.add_attribute(
                            **{
                                'type': mapping['type'],
                                'object_relation': mapping['relation'],
                                'value': value,
                                'to_ids': True
                            })
                    except KeyError:
                        if "x_misp_" in stix_type:
                            attribute_type, relation = stix_type.split(
                                "x_misp_")[1][:-1].split("_")
                            attributes.append({
                                'type': attribute_type,
                                'object_relation': relation,
                                'value': value,
                                'to_ids': True
                            })
            section_uuid = str(uuid.uuid4())
            pe_section.uuid = pe_uuid
            pe.add_reference(section_uuid, 'included-in')
            self.misp_event.add_object(**pe_section)
        self.misp_event.add_object(**pe)
        return attributes, pe_uuid

    def buildExternalDict(self):
        self.fetch_report()
        for o in self.event:
            object_type = o._type
            if object_type in ('relationship', 'report'):
                continue
            if object_type in galaxy_types:
                self.parse_external_galaxy(o)
            elif object_type == 'vulnerability':
                attribute = {'type': 'vulnerability', 'value': o.get('name')}
                if 'description' in o:
                    attribute['comment'] = o.get('description')
                self.misp_event.add_attribute(**attribute)
            elif object_type == 'course-of-action':
                self.parse_course_of_action(o)
            elif object_type == 'indicator':
                pattern = o.get('pattern')
                self.parse_external_pattern(pattern)
                attribute = {
                    'type': 'stix2-pattern',
                    'object_relation': 'stix2-pattern',
                    'value': pattern
                }
                misp_object = {
                    'name': 'stix2-pattern',
                    'meta-category': 'stix2-pattern',
                    'Attribute': [self.version_attribute, attribute]
                }
                self.misp_event.add_object(**misp_object)

    def fetch_report(self):
        reports = []
        for o in self.event:
            if o._type == 'report':
                reports.append(o)
        if len(reports) == 1:
            self.report = reports[0]
            self.parse_report()

    def parse_external_galaxy(self, o):
        galaxy = {'name': galaxy_types[o._type]}
        if 'kill_chain_phases' in o:
            galaxy['type'] = o['kill_chain_phases'][0].get('phase_name')
        cluster = defaultdict(dict)
        cluster['value'] = o.get('name')
        cluster['description'] = o.get('description')
        if 'aliases' in o:
            aliases = []
            for a in o.get('aliases'):
                aliases.append(a)
            cluster['meta']['synonyms'] = aliases
        galaxy['GalaxyCluster'] = [cluster]
        self.misp_event['Galaxy'].append(galaxy)

    def parse_external_pattern(self, pattern):
        if ' OR ' in pattern and ' AND ' not in pattern:
            pattern = pattern.split('OR')
            for p in pattern:
                attribute = self.attribute_from_external_pattern(p)
                self.misp_event.add_attribute(**attribute)
        elif ' OR ' not in pattern and ' LIKE ' not in pattern:
            pattern = pattern.split('AND')
            if len(pattern) == 1:
                attribute = self.attribute_from_external_pattern(pattern[0])
                self.misp_event.add_attribute(**attribute)

    @staticmethod
    def attribute_from_external_pattern(pattern):
        pattern_type, pattern_value = pattern.split(' = ')
        pattern_type, pattern_value = pattern_type[1:].strip(
        ), pattern_value[1:-2].strip()
        stix_type, value_type = pattern_type.split(':')
        if 'hashes' in value_type and 'x509' not in stix_type:
            h_type = value_type.split('.')[1]
            return {'type': h_type, 'value': pattern_value}
        # Might cause some issues, need more examples to test
        return {
            'type':
            external_pattern_mapping[stix_type][value_type].get('type'),
            'value': pattern_value
        }

    def set_distribution(self):
        for attribute in self.misp_event.attributes:
            attribute.distribution = self.__attribute_distribution
        for misp_object in self.misp_event.objects:
            misp_object.distribution = self.__attribute_distribution
            for attribute in misp_object.attributes:
                attribute.distribution = self.__attribute_distribution

    def saveFile(self):
        eventDict = self.misp_event.to_json()
        outputfile = '{}.stix2'.format(self.filename)
        with open(outputfile, 'w') as f:
            f.write(eventDict)

    @staticmethod
    def getTimestampfromDate(stix_date):
        try:
            return int(stix_date.timestamp())
        except AttributeError:
            return int(
                time.mktime(
                    time.strptime(
                        stix_date.split('+')[0], "%Y-%m-%d %H:%M:%S")))

    @staticmethod
    def get_misp_type(labels):
        return labels[0].split('=')[1][1:-1]

    @staticmethod
    def get_misp_category(labels):
        return labels[1].split('=')[1][1:-1]

    @staticmethod
    def parse_observable(observable, attribute_type):
        return misp_types_mapping[attribute_type](observable, attribute_type)

    @staticmethod
    def parse_pattern(pattern):
        if ' AND ' in pattern:
            pattern_parts = pattern.split(' AND ')
            if len(pattern_parts) == 3:
                _, value1 = pattern_parts[2].split(' = ')
                _, value2 = pattern_parts[0].split(' = ')
                return '{}|{}'.format(value1[1:-2], value2[1:-1])
            else:
                _, value1 = pattern_parts[0].split(' = ')
                _, value2 = pattern_parts[1].split(' = ')
                if value1 in ("'ipv4-addr'", "'ipv6-addr'"):
                    return value2[1:-2]
                return '{}|{}'.format(value1[1:-1], value2[1:-2])
        else:
            return pattern.split(' = ')[1][1:-2]

    def parse_pattern_with_data(self, pattern):
        if 'artifact:payload_bin' not in pattern:
            return self.parse_pattern(pattern)
        pattern_parts = pattern.split(' AND ')
        if len(pattern_parts) == 3:
            filename = pattern_parts[0].split(' = ')[1]
            md5 = pattern_parts[1].split(' = ')[1]
            return "{}|{}".format(
                filename[1:-1],
                md5[1:-1]), pattern_parts[2].split(' = ')[1][1:-2]
        return pattern_parts[0].split(' = ')[1][1:-1], pattern_parts[1].split(
            ' = ')[1][1:-2]
Beispiel #57
0
 def setUp(self):
     self.maxDiff = None
     self.mispevent = MISPEvent()
Beispiel #58
0
def check_hashes(self):
    if self.offline_mode:
        self.log('error', 'Offline mode, unable to query VirusTotal')
        return
    event_id = self._get_eventid()
    if event_id is None:
        return
    event = self.misp.get(event_id)
    if self._has_error_message(event):
        return

    misp_event = MISPEvent()
    misp_event.load(event)
    event_hashes = []
    sample_hashes = []
    base_new_attributes = {}
    for a in misp_event.attributes:
        h = None
        if a.type in ('md5', 'sha1', 'sha256'):
            h = a.value
            event_hashes.append(h)
        elif a.type in ('filename|md5', 'filename|sha1', 'filename|sha256', 'malware-sample'):
            h = a.value.split('|')[1]
            event_hashes.append(h)
        if h is not None:
            base_new_attributes[h] = {"category": a.category,
                                      "comment": '{} - Xchecked via VT: {}'.format(a.comment, h),
                                      "to_ids": a.to_ids,
                                      "Tag": a.Tag,
                                      "distribution": a.distribution}

    unk_vt_hashes = []
    vt_request = {'apikey': cfg.virustotal.virustotal_key}
    # Make sure to start getting reports for the longest possible hashes (reduce risks of collisions)
    hashes_to_check = sorted(event_hashes, key=len)
    original_attributes = len(misp_event.attributes)
    if cfg.virustotal.virustotal_has_private_key is False:
        quota = 4
        timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)

    while len(hashes_to_check) > 0:
        vt_request['resource'] = hashes_to_check.pop()
        try:
            response = requests.post(cfg.misp.misp_vturl, data=vt_request, proxies=cfg.virustotal.proxies)
        except requests.ConnectionError:
            self.log('error', 'Failed to connect to VT for {}'.format(vt_request['resource']))
            return
        if response.status_code == 403:
            self.log('error', 'This command requires virustotal API key')
            self.log('error', 'Please check that your key have the right permissions')
            return
        try:
            result = response.json()
        except:
            self.log('error', 'Unable to get the report of {}'.format(vt_request['resource']))
            continue
        if result['response_code'] == 1:
            md5 = result['md5']
            sha1 = result['sha1']
            sha256 = result['sha256']
            hashes_to_check = [eh for eh in hashes_to_check if eh not in (md5, sha1, sha256)]
            link = [False, result['permalink']]
            # Do not re-add a link
            for a in misp_event.attributes:
                if a.value == link[1]:
                    link[0] = True
            if md5 in sample_hashes:
                self.log('success', 'Sample available in MISP:')
            else:
                self.log('success', 'Sample available in VT:')
            if self.args.populate:
                misp_event = self._prepare_attributes(md5, sha1, sha256, link, base_new_attributes, event_hashes, sample_hashes, misp_event)
            self.log('item', '{}\n\t{}\n\t{}\n\t{}'.format(link[1], md5, sha1, sha256))
            if cfg.virustotal.virustotal_has_private_key is False:
                if quota > 0:
                    quota -= 1
                else:
                    waiting_time = (timeout - datetime.datetime.now()).seconds
                    if waiting_time > 0:
                        self.log('warning', 'No private API key, 4 queries/min is the limit. Waiting for {} seconds.'.format(waiting_time))
                        time.sleep(waiting_time)
                    quota = 4
                    timeout = datetime.datetime.now() + datetime.timedelta(minutes=1)
        else:
            unk_vt_hashes.append(vt_request['resource'])

    if self.args.populate:
        self.__populate(misp_event, original_attributes)
    if len(unk_vt_hashes) > 0:
        self.log('error', 'Unknown on VT:')
        for h in unk_vt_hashes:
            self.log('item', '{}'.format(h))
Beispiel #59
0
    def misp_send(self, strMISPEventID, strInput, strInfo, strUsername):
        # Establish communication with MISP
        # event = MISPEvent()
        # event.info = 'Test event'
        # event.analysis = 0
        # event.distribution = 3
        # event.threat_level_id = 2

        # event.add_attribute('md5', '678ff97bf16d8e1c95679c4681834c41')
        # #<add more attributes>

        # self.misp.add_event(event)

        # exit()

        try:
            objects = []
            #get comments and tags from string input
            str_comment, tags = self.get_comm_and_tags(strInput)
            print(tags)
            if tags == None:
                self.misp_logger.info('Irate not in Tags: %s equals None' %
                                      tags)
                response = None
                return response
            #setup misp objects
            mispobj_email = MISPObject(name="email")
            mispobj_file = MISPObject(name="file")
            mispobj_files = {}
            mispobj_domainip = MISPObject(name="domain-ip")
            url_no = 0
            file_no = 0
            mispobj_urls = {}

            #process input
            for line in strInput.splitlines():
                if ("domain:" in line.lower()
                    ):  #Catch domain and add to domain/IP object
                    mispobj_domainip = MISPObject(name="domain-ip")
                    vals = line.split(":", 1)
                    mispobj_domainip.add_attribute("domain",
                                                   value=vals[1].strip(),
                                                   comment=str_comment)
                    objects.append(mispobj_domainip)
                elif ("ip:" in line.lower()) or ("ip-dst:" in line.lower(
                )) or ("ip-src:"
                       in line.lower()):  #Catch IP and add to domain/IP object
                    if "domain:" in strInput.splitlines():
                        mispobj_domainip = MISPObject(name="domain-ip")
                        vals = line.split(":", 1)
                        mispobj_domainip.add_attribute("ip",
                                                       value=vals[1].strip(),
                                                       comment=str_comment)
                        objects.append(mispobj_domainip)
                    else:
                        mispobj_network_connection = MISPObject(
                            name="network-connection")
                        vals = line.split(":", 1)
                        if ("ip:" in line.lower()) or ("ip-dst:"
                                                       in line.lower()):
                            mispobj_network_connection.add_attribute(
                                "ip-dst",
                                type="ip-dst",
                                value=vals[1].strip(),
                                comment=str_comment)
                        else:
                            mispobj_network_connection.add_attribute(
                                "ip-src",
                                type="ip-src",
                                value=vals[1].strip(),
                                comment=str_comment)
                        objects.append(mispobj_network_connection)

                elif ("source-email:"
                      in line.lower()) or ("email-source" in line.lower()) or (
                          "from:" in line.lower()
                      ):  #Catch email and add to email object
                    vals = line.split(":", 1)
                    mispobj_email.add_attribute("from",
                                                value=vals[1].strip(),
                                                comment=str_comment)
                elif ("url:" in line.lower()) or (
                    ('kit:' in line.lower() or ('creds:' in line.lower())) and
                    (('hxxp' in line.lower()) or ('http' in line.lower()))
                ):  #Catch URL and add to URL object
                    vals = line.split(":", 1)
                    url = vals[1].strip()
                    url = refang(url)
                    parsed = urlparse(url)
                    mispobj_url = MISPObject(name="url")
                    mispobj_url.add_attribute("url",
                                              value=parsed.geturl(),
                                              category="Payload delivery",
                                              comment=str_comment)
                    if parsed.hostname:
                        mispobj_url.add_attribute("host",
                                                  value=parsed.hostname,
                                                  comment=str_comment)
                    if parsed.scheme:
                        mispobj_url.add_attribute("scheme",
                                                  value=parsed.scheme,
                                                  comment=str_comment)
                    if parsed.port:
                        mispobj_url.add_attribute("port",
                                                  value=parsed.port,
                                                  comment=str_comment)
                    mispobj_urls[url_no] = mispobj_url
                    url_no += 1

                #Catch different hashes and add to file object
                elif ("sha1:" in line.lower()) or ("SHA1:" in line):
                    vals = line.split(":", 1)
                    mispobj_file.add_attribute("sha1",
                                               value=vals[1].strip(),
                                               comment=str_comment)
                elif ("sha256:" in line.lower()) or ("SHA256:" in line):
                    vals = line.split(":", 1)
                    mispobj_file.add_attribute("sha256",
                                               value=vals[1].strip(),
                                               comment=str_comment)
                elif ("md5:" in line.lower()) or ("MD5:" in line):
                    vals = line.split(":", 1)
                    mispobj_file.add_attribute("md5",
                                               value=vals[1].strip(),
                                               comment=str_comment)
                elif (
                        "subject:" in line.lower()
                ):  #or ("subject:" in line): #Catch subject and add to email object
                    self.misp_logger.info('adding subject')
                    vals = line.split(":", 1)
                    mispobj_email.add_attribute("subject",
                                                value=vals[1].strip(),
                                                comment=str_comment)
                elif ("hash|filename:" in line.lower()
                      ):  #catch hash|filename pair and add to file object
                    vals = line.split(":", 1)
                    val = vals[1].split("|")
                    l_hash = val[0]
                    l_filename = val[1]
                    l_mispobj_file = MISPObject(name="file")
                    if len(re.findall(r"\b[a-fA-F\d]{32}\b", l_hash)) > 0:
                        l_mispobj_file.add_attribute("md5",
                                                     value=l_hash.strip(),
                                                     comment=str_comment)
                        l_mispobj_file.add_attribute("filename",
                                                     value=l_filename.strip(),
                                                     comment=str_comment)
                        mispobj_files[file_no] = l_mispobj_file
                    elif len(re.findall(r'\b[0-9a-f]{40}\b', l_hash)) > 0:
                        l_mispobj_file.add_attribute("sha1",
                                                     value=l_hash.strip(),
                                                     comment=str_comment)
                        l_mispobj_file.add_attribute("filename",
                                                     value=l_filename.strip(),
                                                     comment=str_comment)
                        mispobj_files[file_no] = l_mispobj_file
                    elif len(re.findall(r'\b[A-Fa-f0-9]{64}\b', l_hash)) > 0:
                        l_mispobj_file.add_attribute("sha256",
                                                     value=l_hash.strip(),
                                                     comment=str_comment)
                        l_mispobj_file.add_attribute("filename",
                                                     value=l_filename.strip(),
                                                     comment=str_comment)
                        mispobj_files[file_no] = l_mispobj_file
                    file_no += 1

            #add all misp objects to List to be processed and submitted to MISP server as one.
            if len(mispobj_file.attributes) > 0:
                objects.append(mispobj_file)
            if len(mispobj_email.attributes) > 0:
                objects.append(mispobj_email)

            for u_key, u_value in mispobj_urls.items():
                if len(u_value.attributes) > 0:
                    objects.append(u_value)
            for f_key, f_value in mispobj_files.items():
                if len(f_value.attributes) > 0:
                    objects.append(f_value)
            # Update timestamp and event

        except Exception as e:
            error = traceback.format_exc()
            response = "Error occured when converting string to misp objects:\n %s" % error
            self.misp_logger.error(response)
            return response

        if self.check_object_length(objects) != True:
            self.misp_logger.error(
                'Input from %s did not contain accepted tags.\n Input: \n%s' %
                (strUsername, strInput))
            return "Error in the tags you entered. Please see the guide for accepted tags."

        try:
            # self.misp_logger.error(dir(self.misp))
            misp_event = MISPEvent()
            misp_event.info = strInfo
            misp_event.distribution = 0
            misp_event.analysis = 2
            misp_event.threat_level_id = 3
            # event.add_attribute('md5', '678ff97bf16d8e1c95679c4681834c41')
            #event = self.misp.new_event(info=strInfo, distribution='0', analysis='2', threat_level_id='3', published=False)
            #misp_event = MISPEvent()
            #misp_event.load(event)
            add = self.misp.add_event(misp_event)
            self.misp_logger.info("Added event %s" % add)
            a, b = self.submit_to_misp(self.misp, misp_event, objects)
            for tag in tags:
                self.misp.tag(misp_event.uuid, tag)
            #self.misp.add_internal_comment(misp_event.id, reference="Author: " + strUsername, comment=str_comment)
            ccc = self.misp.publish(misp_event, alert=False)
            self.misp_logger.info(ccc)
            misp_event = self.misp.get_event(misp_event)
            response = misp_event
            #for response in misp_event:
            if ('errors' in response and response['errors'] != None):
                return ("Submission error: " + repr(response['errors']))
            else:
                if response['Event']['RelatedEvent']:
                    e_related = ""
                    for each in response['Event']['RelatedEvent']:
                        e_related = e_related + each['Event']['id'] + ", "
                    return "Created ID: " + str(
                        response['Event']
                        ['id']) + "\nRelated Events: " + ''.join(e_related)
                else:
                    return "Created ID: " + str(response['Event']['id'])

        except Exception as e:
            error = traceback.format_exc()
            response = "Error occured when submitting to misp:\n %s" % error
            self.misp_logger.error(response)
            return response
Beispiel #60
0
class ReportGenerator():
    def __init__(self, profile="daily_report"):
        self.taxonomies = Taxonomies()
        self.report = ''
        profile_name = "profiles.{}".format(profile)
        self.template = importlib.import_module(name=profile_name)

    def from_remote(self, event_id):
        from pymisp import PyMISP
        from keys import misp_url, misp_key, misp_verifycert
        misp = PyMISP(misp_url, misp_key, misp_verifycert)
        result = misp.get(event_id)
        self.misp_event = MISPEvent()
        self.misp_event.load(result)

    def from_file(self, path):
        self.misp_event = MISPEvent()
        self.misp_event.load_file(path)

    def attributes(self):
        if not self.misp_event.attributes:
            return ''
        list_attributes = []
        for attribute in self.misp_event.attributes:
            if attribute.type in self.template.types_to_attach:
                list_attributes.append("* {}".format(defang(attribute.value)))
        for obj in self.misp_event.Object:
            if obj.name in self.template.objects_to_attach:
                for attribute in obj.Attribute:
                    if attribute.type in self.template.types_to_attach:
                        list_attributes.append("* {}".format(defang(attribute.value)))
        return self.template.attributes.format(list_attributes="\n".join(list_attributes))

    def _get_tag_info(self, machinetag):
        return self.taxonomies.revert_machinetag(machinetag)

    def report_headers(self):
        content = {'org_name': 'name',
                   'date': date.today().isoformat()}
        self.report += self.template.headers.format(**content)

    def event_level_tags(self):
        if not self.misp_event.Tag:
            return ''
        for tag in self.misp_event.Tag:
            # Only look for TLP for now
            if tag['name'].startswith('tlp'):
                tax, predicate = self._get_tag_info(tag['name'])
                return self.template.event_level_tags.format(value=predicate.predicate.upper(), expanded=predicate.expanded)

    def title(self):
        internal_id = ''
        summary = ''
        # Get internal refs for report
        for obj in self.misp_event.Object:
            if obj.name != 'report':
                continue
            for a in obj.Attribute:
                if a.object_relation == 'case-number':
                    internal_id = a.value
                if a.object_relation == 'summary':
                    summary = a.value

        return self.template.title.format(internal_id=internal_id, title=self.misp_event.info,
                                          summary=summary)

    def asciidoc(self, lang='en'):
        self.report += self.title()
        self.report += self.event_level_tags()
        self.report += self.attributes()