def create_daily_event(self): today = str(datetime.date.today()) event_dict = { 'id': len(self.manifest) + 1, 'Tag': settings.Tag, 'info': self.daily_event_name.format(today), 'analysis': settings.analysis, # [0-2] 'threat_level_id': settings.threat_level_id, # [1-4] 'published': settings.published, 'date': today } event = MISPEvent() event.from_dict(**event_dict) # reference org org = MISPOrganisation() org.name = settings.org_name org.uuid = settings.org_uuid event.Orgc = org # save event on disk self.flush_event(new_event=event) # add event to manifest self.manifest.update(event.manifest) self.save_manifest() return event
def __init__(self, params): self.initial_user_connector = ExpandedPyMISP(params['url'], params['key'], ssl=False, debug=False) # Git pull self.initial_user_connector.update_misp() # Set the default role (id 3 on the VM is normal user) self.initial_user_connector.set_default_role(3) # Restart workers self.initial_user_connector.restart_workers() if not fast_mode: # Load submodules self.initial_user_connector.update_object_templates() self.initial_user_connector.update_galaxies() self.initial_user_connector.update_noticelists() self.initial_user_connector.update_warninglists() self.initial_user_connector.update_taxonomies() self.initial_user_connector.toggle_global_pythonify() # Create organisation organisation = MISPOrganisation() organisation.name = params['orgname'] self.test_org = self.initial_user_connector.add_organisation(organisation) print(self.test_org.name, self.test_org.uuid) # Create Site admin in new org user = MISPUser() user.email = params['email_site_admin'] user.org_id = self.test_org.id user.role_id = 1 # Site admin self.test_site_admin = self.initial_user_connector.add_user(user) self.site_admin_connector = ExpandedPyMISP(params['url'], self.test_site_admin.authkey, ssl=False, debug=False) self.site_admin_connector.toggle_global_pythonify() # Create org admin user = MISPUser() user.email = params['email_admin'] user.org_id = self.test_org.id user.role_id = 2 # Org admin self.test_org_admin = self.site_admin_connector.add_user(user) self.org_admin_connector = ExpandedPyMISP(params['url'], self.test_org_admin.authkey, ssl=False, debug=False) self.org_admin_connector.toggle_global_pythonify() # Create user user = MISPUser() user.email = params['email_user'] user.org_id = self.test_org.id self.test_usr = self.org_admin_connector.add_user(user) self.user_connector = ExpandedPyMISP(params['url'], self.test_usr.authkey, ssl=False, debug=False) self.user_connector.toggle_global_pythonify() # Setup external_baseurl self.site_admin_connector.set_server_setting('MISP.external_baseurl', params['external_baseurl'], force=True) # Setup baseurl self.site_admin_connector.set_server_setting('MISP.baseurl', params['url'], force=True) # Setup host org self.site_admin_connector.set_server_setting('MISP.host_org_id', self.test_org.id) self.external_base_url = params['external_baseurl'] self.sync = [] self.sync_servers = []
def setUpClass(cls): warnings.simplefilter("ignore", ResourceWarning) # Connect as admin cls.admin_misp_connector = PyMISP(url, key) cls.admin_misp_connector.set_server_setting('debug', 1, force=True) cls.admin_misp_connector.global_pythonify = True # Check if admin is really site admin assert cls.admin_misp_connector._current_role.perm_site_admin # Create advanced authkey, so connector will work even after advanced keys are required cls.admin_advanced_authkey = cls.__create_advanced_authkey( cls, cls.admin_misp_connector._current_user.id) cls.admin_misp_connector.key = cls.admin_misp_connector.key + "," + cls.admin_advanced_authkey[ "authkey_raw"] # Creates an org organisation = MISPOrganisation() organisation.name = 'Test Org ' + random() # make name always unique cls.test_org = cls.admin_misp_connector.add_organisation(organisation) check_response(cls.test_org) # Creates org admin org_admin = MISPUser() org_admin.email = 'testorgadmin@user' + random( ) + '.local' # make name always unique org_admin.org_id = cls.test_org.id org_admin.role_id = 2 # Org admin role cls.test_org_admin = cls.admin_misp_connector.add_user(org_admin) check_response(cls.test_org_admin) # Creates advanced auth key for org admin cls.org_admin_advanced_authkey = cls.__create_advanced_authkey( cls, cls.test_org_admin.id) cls.org_admin_misp_connector = PyMISP( url, cls.test_org_admin.authkey + "," + cls.org_admin_advanced_authkey["authkey_raw"]) cls.org_admin_misp_connector.global_pythonify = True # Creates an user cls.test_usr_password = str(uuid.uuid4()) user = MISPUser() user.email = 'testusr@user' + random( ) + '.local' # make name always unique user.org_id = cls.test_org.id user.role_id = 3 # User role user.password = cls.test_usr_password cls.test_usr = cls.admin_misp_connector.add_user(user) check_response(cls.test_usr) # Try to connect as user to check if everything works PyMISP(url, cls.test_usr.authkey) # Check if user can login with given password assert login(url, cls.test_usr.email, cls.test_usr_password)
def setUpClass(cls): cls.maxDiff = None # Connect as admin cls.admin_misp_connector = PyMISP(url, key, ssl=False, debug=False) cls.admin_misp_connector.set_server_setting('debug', 1, force=True) # Creates an org organisation = MISPOrganisation() organisation.name = 'Test Org' cls.test_org = cls.admin_misp_connector.add_organisation(organisation, pythonify=True) # Set the default role (id 3 on the VM) cls.admin_misp_connector.set_default_role(3) # Creates a user user = MISPUser() user.email = '*****@*****.**' user.org_id = cls.test_org.id cls.test_usr = cls.admin_misp_connector.add_user(user, pythonify=True) cls.user_misp_connector = PyMISP(url, cls.test_usr.authkey, ssl=False, debug=True)
def test_feed(self): me = MISPEvent() me.info = 'Test feed' org = MISPOrganisation() org.name = 'TestOrg' org.uuid = '123478' me.Orgc = org me.add_attribute('ip-dst', '8.8.8.8') obj = me.add_object(name='file') obj.add_attributes('filename', *['foo.exe', 'bar.exe']) h = hashlib.new('md5') h.update(b'8.8.8.8') hash_attr_val = h.hexdigest() feed = me.to_feed(with_meta=True) self.assertEqual(feed['Event']['_hashes'][0], hash_attr_val) self.assertEqual(feed['Event']['_manifest'][me.uuid]['info'], 'Test feed') self.assertEqual(len(feed['Event']['Object'][0]['Attribute']), 2)
def poll_taxii(): global f_hashes, f_manifest, f_events results_dict = {} client = create_client( CYBERSAIYAN_FEED_URL, use_https=TAXII_USE_TLS, discovery_path=TAXII_DISCOVERY_PATH ) blocks = client.poll(collection_name=CYBERSAIYAN_COLLECTION_NAME) for block in blocks: content = block.content if content: if type(content) == str: continue elif type(content) == bytes: content = content.decode('utf-8') pkg = STIXPackage.from_xml(StringIO(content)) title = pkg.stix_header.title information_source = pkg.stix_header.information_source.identity.name cs_event = (title, information_source) cs_event_hash = hash(cs_event) db_cursor.execute("SELECT uuid FROM hashes WHERE hash = '%s'" % cs_event_hash) element = db_cursor.fetchone() if element: e_uuid = element[0] else: e_uuid = str(uuid.uuid4()) db_cursor.execute("INSERT INTO hashes VALUES (?,?)", (cs_event_hash,e_uuid,)) if cs_event_hash not in results_dict: results_dict[cs_event_hash] = MISPEvent() m_ev = results_dict[cs_event_hash] m_ev.info = str(pkg.stix_header.description) m_ev.analysis = 0 m_ev.uuid = e_uuid #m_ev.org = "CyberSaiyan" csorg = MISPOrganisation() csorg.name = "CyberSaiyan" csorg.uuid = "8aaa81ed-72ef-4fb1-8e96-fa1bc200faeb" m_ev.orgc = csorg marking = pkg.stix_header.handling.marking tlp = 0 found_tlp = False for m in marking: for struct in m.marking_structures: if struct._XSI_TYPE == "tlpMarking:TLPMarkingStructureType": found_tlp = True tlp = max(TLP[struct.color.lower()], tlp) if tlp == 0 and not found_tlp: tlp = TLP["amber"] m_ev.add_tag("tlp:"+TLP[tlp]) m_ev.add_tag("CyberSaiyan") indicators = pkg.indicators last_ts = utc.localize(datetime.datetime(1970,1,1)) for indicator in indicators: cur_ts = indicator.timestamp if cur_ts > last_ts: last_ts = cur_ts obj = indicator.observable.object_ obj_d = obj.properties.to_dict() attr_type = obj_d["xsi:type"] if attr_type == "AddressObjectType": attr = MISPAttribute() attr.category = "Network activity" attr.type = "ip-dst" attr.value = obj_d["address_value"] attr.disable_correlation = False attr.to_ids = True elif attr_type == "DomainNameObjectType": attr = MISPAttribute() attr.category = "Network activity" attr.type = "domain" attr.value = obj_d["value"] attr.disable_correlation = False attr.to_ids = True elif attr_type == "URIObjectType": attr = MISPAttribute() attr.category = "Network activity" attr.type = "url" attr.value = obj_d["value"] attr.disable_correlation = False attr.to_ids = True elif attr_type == "FileObjectType": hash_type = obj_d["hashes"][0]["type"]["value"].lower() hash_value = obj_d["hashes"][0]["simple_hash_value"] attr = MISPAttribute() attr.category = "Payload delivery" assert hash_type in ('md5', "sha1", "sha224", "sha256", "sha384", "sha512", "ssdeep") attr.type = hash_type attr.value = hash_value attr.disable_correlation = False attr.to_ids = True m_ev.date = last_ts.strftime("%Y-%m-%d") m_ev.attributes.append(attr) db_conn.commit() c_hashes, c_manifest, c_events = list(), dict(), dict() for event in results_dict.values(): e_feed = event.to_feed(with_meta=True).get("Event") c_hashes += [[h, event.uuid] for h in e_feed.pop("_hashes")] c_manifest.update(e_feed.pop('_manifest')) c_events[event.uuid] = e_feed f_hashes, f_manifest, f_events = c_hashes, c_manifest, c_events
from keys import misp_url, misp_key, misp_verifycert, proofpoint_key # initialize PyMISP and set url for Panorama misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert) urlVap = "https://tap-api-v2.proofpoint.com/v2/people/vap?window=30" # Window can be 14, 30, and 90 Days headers = {'Authorization': "Basic " + proofpoint_key} responseVap = requests.request("GET", urlVap, headers=headers) jsonDataVap = json.loads(responseVap.text) for alert in jsonDataVap["users"]: orgc = MISPOrganisation() orgc.name = 'Proofpoint' orgc.id = '#{ORGC.ID}' # organisation id orgc.uuid = '#{ORGC.UUID}' # organisation uuid # initialize and set MISPEvent() event = MISPEvent() event.Orgc = orgc event.info = 'Very Attacked Person ' + jsonDataVap["interval"] event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config event.threat_level_id = 2 # setting this to 0 breaks the integration event.analysis = 0 # Optional, defaults to 0 (initial analysis) totalVapUsers = event.add_attribute('counter', jsonDataVap["totalVapUsers"], comment="Total VAP Users") averageAttackIndex = event.add_attribute('counter',
"async":"yes", "reporttype":"predefined", "reportname":"top-attacks", "key":"#{API_KEY}" # put your Panorama api key here } # try with "..., verify=False)" if you get an SSL error response = requests.request("GET", url, params=querystring) resp_text = response.text json_data = json.loads(json.dumps(xmltodict.parse(resp_text))) # initialize and set MISPOrganisation orgc = MISPOrganisation() orgc.name = 'Palo Alto' orgc.id = '#{ORGC_ID}' # organisation id orgc.uuid = '#{ORGC_UUID}' # organisation uuid # initialize and set MISPEvent() event = MISPEvent() event.Orgc = orgc event.info = json_data['report']['result']['@name'] + " | " + json_data['report']['result']['@range'] event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config event.threat_level_id = 0 # Optional, defaults to MISP.default_event_threat_level in MISP config event.analysis = 0 # Optional, defaults to 0 (initial analysis) event.add_tag('firewall threats') for threatid in json_data['report']['result']['entry']: attribute = event.add_attribute('comment', threatid['threatid']) attribute.comment = threatid['count']
def host_org(self) -> MISPOrganisation: organisation = MISPOrganisation() organisation.name = self.config['admin_orgname'] return self.create_or_update_organisation(organisation)
# convert to millis since epoch to_time = datetime_to_millis(to_time) from_time = datetime_to_millis(from_time) rhisac = "7a33144f-aef3-442b-87d4-dbf70d8afdb0" # TruStar RH-ISAC enclave uuid reports = tru.get_reports(from_time=from_time, to_time=to_time, is_enclave=True, enclave_ids=rhisac) # loop through each trustar report and create MISP event for report in reports: # initialize and set MISPOrganisation() orgc = MISPOrganisation() orgc.name = 'RH-ISAC' orgc.id = '#{ORGC_ID}' # organisation id orgc.uuid = '#{ORGC_UUID}' # organisation uuid # initialize and set MISPEvent() event = MISPEvent() event.Orgc = orgc event.info = report.title event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config event.threat_level_id = 2 # Optional, defaults to MISP.default_event_threat_level in MISP config event.analysis = 0 # Optional, defaults to 0 (initial analysis) # get tags for report for tag in tru.get_enclave_tags(report.id): event.add_tag(tag.name) # get indicators for report
# -*- coding: utf-8 -*- from pathlib import Path from pymisp import MISPEvent, MISPOrganisation, PyMISP from dateutil.parser import parse import json from pymisp.tools import feed_meta_generator from io import BytesIO make_feed = False path = Path('/home/raphael/gits/covid-19-china/data') if make_feed: org = MISPOrganisation() org.name = 'CIRCL' org.uuid = "55f6ea5e-2c60-40e5-964f-47a8950d210f" else: from covid_key import url, key misp = PyMISP(url, key) for p in path.glob('*_json/current_china.json'): d = parse(p.parent.name[:-5]) event = MISPEvent() event.info = f"[{d.isoformat()}] DXY COVID-19 live report" event.date = d event.distribution = 3 event.add_tag('tlp:white') if make_feed: event.orgc = org else:
event_import_info = False json_import = sys.argv[1] event_import_org = sys.argv[2] event_import_uuid = str(uuid.uuid4()) # Unique ID event_import_date = date.today() # Create event with current data event_import_distribution = 2 # Connected # Check if organisation already exist org = MISPOrganisation() try: org.id = api.get_organisation(event_import_org, pythonify=True).id except: # We need to create a new one org_new = MISPOrganisation() org_new.name = event_import_org org_new.uuid = str(uuid.uuid4()) org_new.type = "CSIRT" org_new.sector = "Government" org.id = api.add_organisation(org_new, pythonify=True).id # Create the MISP event by loading the JSON file # This will not add the attributes, but does add the event tags and galaxies # We also add a random UUID for uniqueness event = MISPEvent() event.load_file(json_import) event.uuid = event_import_uuid if not event_import_info: event_import_info = event.info event.info = event_import_info event.date = event_import_date
def test_Organisations(self): """ This test creats new organisations from the org.json file and adds user from the user.json file to it. After all organisations and user are created the tests removes them and checks if they are correctly removed """ ### Create organisations from org.json file org_list = readInFile("samples/org.json") tested_keys = [ 'name', 'description', 'nationality', 'sector', 'uuid', 'contacts' ] for item in org_list: org = MISPOrganisation() org.name = org_list[item]['name'] org.description = org_list[item]['description'] org.nationality = org_list[item]['nationality'] org.sector = org_list[item]['sector'] org.uuid = org_list[item]['uuid'] org.contacts = org_list[item]['contacts'] #org.local = org_list[item]['local'] logging.info("OrgManagement - try to create organization \"" + org_list[item]['name'] + "\"") response = self._misp.add_organisation(org, pythonify=True) self.assertTrue( org_list[item]['uuid'] in response.uuid, msg="The created organisation has no or a wrong UUID") self.assertTrue( org_list[item]['name'] in response.name, msg="The created organisation has no or a wrong name") self.assertTrue( org_list[item]['description'] in response.description, msg="The created organisation has no or a wrong description") self.assertTrue( org_list[item]['nationality'] in response.nationality, msg="The created organisation has no or a wrong nationality") self.assertTrue( response.local, msg= "The created organisation is not a local organisation but should be a local organisation" ) self.assertTrue( org_list[item]['sector'] in response.sector, msg="The created organisation has no or a wrong sector") response = self._misp.organisations(scope="local", pythonify=True) logging.info( "OrgManagement - check if the admin and both test organisations exist" ) self.assertGreaterEqual( len(response), 3, "MISP responded with less then 3 existing organisations - there shold exactly be 3" ) ### Add new user from the user.json file list_user = readInFile("samples/user.json") users = self._misp.users(pythonify=True) for item in list_user: for org in response: if org.name in list_user[item]['org_id']: logging.info("OrgManagement - try to add user \"" + list_user[item]['email'] + "\"") usr = MISPUser() usr.email = list_user[item]['email'] usr.org_id = org.id usr.role_id = list_user[item]['role_id'] usr_response = self._misp.add_user(usr, pythonify=True) # legnth of regular authkeys is 40 chars or longer self.assertTrue( usr_response.email in list_user[item]['email'], msg="The created users has no or a wrong email") self.assertTrue( usr_response.role_id in list_user[item]['role_id'], msg="The created users has no or a wrong role id") self.assertGreaterEqual( len(usr_response.authkey), 40, msg= "MISP responded with a wrong authkey - should be exactly 40 chars" ) ### An authentication test could be inserted here ### An user change role test could be inserted here logging.info( "OrgManagement - check if all user where created successfully") response = self._misp.users(pythonify=True) self.assertGreaterEqual( len(response), len(list_user), msg= "MISP responded with a wrong number of users - it seems that not all users could be created." ) for item in response: if item.org_id not in '1' or item.id not in '1': logging.info("OrgManagement - try to delete user \"" + item.email + "\"") usr_response = self._misp.delete_user(item) self.assertTrue("User deleted" in usr_response['message'], msg="User could ne be deleted") pass logging.info( "OrgManagement - check if user list now only contains the admin user" ) response = self._misp.users(pythonify=True) self.assertEqual( len(response), 1, "MISP responded with a wrong number of users - it seems that not all users could be deleted." ) ### Remove organisations response = self._misp.organisations(pythonify=True) for item in response: if item.id not in "1": logging.info("Try to remove organization: \"" + item.name + "\"") org_response = self._misp.delete_organisation(item) self.assertTrue( 'deleted' in org_response['message'], msg="Organisations could not be deleted from MISP") pass response = self._misp.organisations(pythonify=True) logging.info("OrgManagement - check if only admin org exist") self.assertEqual( len(response), 1, msg= "MISP responded with a wrong number of organisations - it seems that not all organisations could be deleted." )
if __name__ == '__main__': parser = argparse.ArgumentParser(description='Add organizations from a CSV file') parser.add_argument("-c", "--csv-import", required=True, help="The CSV file containing the organizations. Format 'orgname,nationality,sector,type,contacts,uuid,local,sharingroup_uuid'") args = parser.parse_args() misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert) # CSV format # orgname,nationality,sector,type,contacts,uuid,local,sharingroup with open(args.csv_import) as csv_file: count_orgs = 0 csv_reader = csv.reader(csv_file, delimiter=',') for row in csv_reader: org = MISPOrganisation() org.name = row[0] print("Process {}".format(org.name)) org.nationality = row[1] org.sector = row[2] org.type = row[3] org.contacts = row[4] org.uuid = row[5] org.local = row[6] add_org = misp.add_organisation(org, pythonify=True) if 'errors' in add_org: print(add_org['errors']) else: count_orgs = count_orgs + 1 org_uuid = add_org.uuid
def __init__(self, misp_instance_dir: Path, secure_connection: bool): with (misp_instance_dir / 'config.json').open() as f: self.instance_config = json.load(f) print('Initialize', self.instance_config['admin_orgname']) self.secure_connection = secure_connection self.synchronisations = {} self.name = self.instance_config['admin_orgname'] # NOTE: never use that user again after initial config. initial_user_connector = PyMISP(self.instance_config['baseurl'], self.instance_config['admin_key'], ssl=self.secure_connection, debug=False) # Set the default role (id 3 is normal user) initial_user_connector.set_default_role(3) initial_user_connector.toggle_global_pythonify() self.baseurl = self.instance_config['baseurl'] self.external_baseurl = self.instance_config['external_baseurl'] # Create organisation organisation = MISPOrganisation() organisation.name = self.instance_config['admin_orgname'] self.host_org = initial_user_connector.add_organisation(organisation) if not isinstance(self.host_org, MISPOrganisation): # The organisation is probably already there organisations = initial_user_connector.organisations() for organisation in organisations: if organisation.name == self.instance_config['admin_orgname']: self.host_org = organisation break else: raise Exception('Unable to find admin organisation') # Create Site admin in new org user = MISPUser() user.email = self.instance_config['email_site_admin'] user.org_id = self.host_org.id user.role_id = 1 # Site admin self.host_site_admin = initial_user_connector.add_user(user) if not isinstance(self.host_site_admin, MISPUser): users = initial_user_connector.users() for user in users: if user.email == self.instance_config['email_site_admin']: self.host_site_admin = user break else: raise Exception('Unable to find admin user') self.site_admin_connector = PyMISP(self.baseurl, self.host_site_admin.authkey, ssl=self.secure_connection, debug=False) self.site_admin_connector.toggle_global_pythonify() # Setup external_baseurl self.site_admin_connector.set_server_setting('MISP.external_baseurl', self.external_baseurl, force=True) # Setup baseurl self.site_admin_connector.set_server_setting('MISP.baseurl', self.baseurl, force=True) # Setup host org self.site_admin_connector.set_server_setting('MISP.host_org_id', self.host_org.id) # create other useful users self.orgadmin = self.create_user( self.instance_config['email_orgadmin'], 2) self.user = self.create_user(self.instance_config['email_user'], 3) # And connectors self.org_admin_connector = PyMISP(self.baseurl, self.orgadmin.authkey, ssl=self.secure_connection, debug=False) self.org_admin_connector.toggle_global_pythonify() self.user_connector = PyMISP(self.baseurl, self.user.authkey, ssl=self.secure_connection, debug=False) self.user_connector.toggle_global_pythonify()