def create_daily_event(self): today = str(datetime.date.today()) event_dict = { 'id': len(self.manifest) + 1, 'Tag': settings.Tag, 'info': self.daily_event_name.format(today), 'analysis': settings.analysis, # [0-2] 'threat_level_id': settings.threat_level_id, # [1-4] 'published': settings.published, 'date': today } event = MISPEvent() event.from_dict(**event_dict) # reference org org = MISPOrganisation() org.name = settings.org_name org.uuid = settings.org_uuid event.Orgc = org # save event on disk self.flush_event(new_event=event) # add event to manifest self.manifest.update(event.manifest) self.save_manifest() return event
def test_feed(self): me = MISPEvent() me.info = 'Test feed' org = MISPOrganisation() org.name = 'TestOrg' org.uuid = '123478' me.Orgc = org me.add_attribute('ip-dst', '8.8.8.8') obj = me.add_object(name='file') obj.add_attributes('filename', *['foo.exe', 'bar.exe']) h = hashlib.new('md5') h.update(b'8.8.8.8') hash_attr_val = h.hexdigest() feed = me.to_feed(with_meta=True) self.assertEqual(feed['Event']['_hashes'][0], hash_attr_val) self.assertEqual(feed['Event']['_manifest'][me.uuid]['info'], 'Test feed') self.assertEqual(len(feed['Event']['Object'][0]['Attribute']), 2)
def poll_taxii(): global f_hashes, f_manifest, f_events results_dict = {} client = create_client( CYBERSAIYAN_FEED_URL, use_https=TAXII_USE_TLS, discovery_path=TAXII_DISCOVERY_PATH ) blocks = client.poll(collection_name=CYBERSAIYAN_COLLECTION_NAME) for block in blocks: content = block.content if content: if type(content) == str: continue elif type(content) == bytes: content = content.decode('utf-8') pkg = STIXPackage.from_xml(StringIO(content)) title = pkg.stix_header.title information_source = pkg.stix_header.information_source.identity.name cs_event = (title, information_source) cs_event_hash = hash(cs_event) db_cursor.execute("SELECT uuid FROM hashes WHERE hash = '%s'" % cs_event_hash) element = db_cursor.fetchone() if element: e_uuid = element[0] else: e_uuid = str(uuid.uuid4()) db_cursor.execute("INSERT INTO hashes VALUES (?,?)", (cs_event_hash,e_uuid,)) if cs_event_hash not in results_dict: results_dict[cs_event_hash] = MISPEvent() m_ev = results_dict[cs_event_hash] m_ev.info = str(pkg.stix_header.description) m_ev.analysis = 0 m_ev.uuid = e_uuid #m_ev.org = "CyberSaiyan" csorg = MISPOrganisation() csorg.name = "CyberSaiyan" csorg.uuid = "8aaa81ed-72ef-4fb1-8e96-fa1bc200faeb" m_ev.orgc = csorg marking = pkg.stix_header.handling.marking tlp = 0 found_tlp = False for m in marking: for struct in m.marking_structures: if struct._XSI_TYPE == "tlpMarking:TLPMarkingStructureType": found_tlp = True tlp = max(TLP[struct.color.lower()], tlp) if tlp == 0 and not found_tlp: tlp = TLP["amber"] m_ev.add_tag("tlp:"+TLP[tlp]) m_ev.add_tag("CyberSaiyan") indicators = pkg.indicators last_ts = utc.localize(datetime.datetime(1970,1,1)) for indicator in indicators: cur_ts = indicator.timestamp if cur_ts > last_ts: last_ts = cur_ts obj = indicator.observable.object_ obj_d = obj.properties.to_dict() attr_type = obj_d["xsi:type"] if attr_type == "AddressObjectType": attr = MISPAttribute() attr.category = "Network activity" attr.type = "ip-dst" attr.value = obj_d["address_value"] attr.disable_correlation = False attr.to_ids = True elif attr_type == "DomainNameObjectType": attr = MISPAttribute() attr.category = "Network activity" attr.type = "domain" attr.value = obj_d["value"] attr.disable_correlation = False attr.to_ids = True elif attr_type == "URIObjectType": attr = MISPAttribute() attr.category = "Network activity" attr.type = "url" attr.value = obj_d["value"] attr.disable_correlation = False attr.to_ids = True elif attr_type == "FileObjectType": hash_type = obj_d["hashes"][0]["type"]["value"].lower() hash_value = obj_d["hashes"][0]["simple_hash_value"] attr = MISPAttribute() attr.category = "Payload delivery" assert hash_type in ('md5', "sha1", "sha224", "sha256", "sha384", "sha512", "ssdeep") attr.type = hash_type attr.value = hash_value attr.disable_correlation = False attr.to_ids = True m_ev.date = last_ts.strftime("%Y-%m-%d") m_ev.attributes.append(attr) db_conn.commit() c_hashes, c_manifest, c_events = list(), dict(), dict() for event in results_dict.values(): e_feed = event.to_feed(with_meta=True).get("Event") c_hashes += [[h, event.uuid] for h in e_feed.pop("_hashes")] c_manifest.update(e_feed.pop('_manifest')) c_events[event.uuid] = e_feed f_hashes, f_manifest, f_events = c_hashes, c_manifest, c_events
# initialize PyMISP and set url for Panorama misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert) urlVap = "https://tap-api-v2.proofpoint.com/v2/people/vap?window=30" # Window can be 14, 30, and 90 Days headers = {'Authorization': "Basic " + proofpoint_key} responseVap = requests.request("GET", urlVap, headers=headers) jsonDataVap = json.loads(responseVap.text) for alert in jsonDataVap["users"]: orgc = MISPOrganisation() orgc.name = 'Proofpoint' orgc.id = '#{ORGC.ID}' # organisation id orgc.uuid = '#{ORGC.UUID}' # organisation uuid # initialize and set MISPEvent() event = MISPEvent() event.Orgc = orgc event.info = 'Very Attacked Person ' + jsonDataVap["interval"] event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config event.threat_level_id = 2 # setting this to 0 breaks the integration event.analysis = 0 # Optional, defaults to 0 (initial analysis) totalVapUsers = event.add_attribute('counter', jsonDataVap["totalVapUsers"], comment="Total VAP Users") averageAttackIndex = event.add_attribute('counter', jsonDataVap["averageAttackIndex"], comment="Average Attack Count")
from pathlib import Path from pymisp import MISPEvent, MISPOrganisation, PyMISP from dateutil.parser import parse import json from pymisp.tools import feed_meta_generator from io import BytesIO make_feed = False path = Path('/home/raphael/gits/covid-19-china/data') if make_feed: org = MISPOrganisation() org.name = 'CIRCL' org.uuid = "55f6ea5e-2c60-40e5-964f-47a8950d210f" else: from covid_key import url, key misp = PyMISP(url, key) for p in path.glob('*_json/current_china.json'): d = parse(p.parent.name[:-5]) event = MISPEvent() event.info = f"[{d.isoformat()}] DXY COVID-19 live report" event.date = d event.distribution = 3 event.add_tag('tlp:white') if make_feed: event.orgc = org else: e = misp.search(eventinfo=event.info, metadata=True, pythonify=True)
json_import = sys.argv[1] event_import_org = sys.argv[2] event_import_uuid = str(uuid.uuid4()) # Unique ID event_import_date = date.today() # Create event with current data event_import_distribution = 2 # Connected # Check if organisation already exist org = MISPOrganisation() try: org.id = api.get_organisation(event_import_org, pythonify=True).id except: # We need to create a new one org_new = MISPOrganisation() org_new.name = event_import_org org_new.uuid = str(uuid.uuid4()) org_new.type = "CSIRT" org_new.sector = "Government" org.id = api.add_organisation(org_new, pythonify=True).id # Create the MISP event by loading the JSON file # This will not add the attributes, but does add the event tags and galaxies # We also add a random UUID for uniqueness event = MISPEvent() event.load_file(json_import) event.uuid = event_import_uuid if not event_import_info: event_import_info = event.info event.info = event_import_info event.date = event_import_date event.distribution = event_import_distribution
def test_Organisations(self): """ This test creats new organisations from the org.json file and adds user from the user.json file to it. After all organisations and user are created the tests removes them and checks if they are correctly removed """ ### Create organisations from org.json file org_list = readInFile("samples/org.json") tested_keys = [ 'name', 'description', 'nationality', 'sector', 'uuid', 'contacts' ] for item in org_list: org = MISPOrganisation() org.name = org_list[item]['name'] org.description = org_list[item]['description'] org.nationality = org_list[item]['nationality'] org.sector = org_list[item]['sector'] org.uuid = org_list[item]['uuid'] org.contacts = org_list[item]['contacts'] #org.local = org_list[item]['local'] logging.info("OrgManagement - try to create organization \"" + org_list[item]['name'] + "\"") response = self._misp.add_organisation(org, pythonify=True) self.assertTrue( org_list[item]['uuid'] in response.uuid, msg="The created organisation has no or a wrong UUID") self.assertTrue( org_list[item]['name'] in response.name, msg="The created organisation has no or a wrong name") self.assertTrue( org_list[item]['description'] in response.description, msg="The created organisation has no or a wrong description") self.assertTrue( org_list[item]['nationality'] in response.nationality, msg="The created organisation has no or a wrong nationality") self.assertTrue( response.local, msg= "The created organisation is not a local organisation but should be a local organisation" ) self.assertTrue( org_list[item]['sector'] in response.sector, msg="The created organisation has no or a wrong sector") response = self._misp.organisations(scope="local", pythonify=True) logging.info( "OrgManagement - check if the admin and both test organisations exist" ) self.assertGreaterEqual( len(response), 3, "MISP responded with less then 3 existing organisations - there shold exactly be 3" ) ### Add new user from the user.json file list_user = readInFile("samples/user.json") users = self._misp.users(pythonify=True) for item in list_user: for org in response: if org.name in list_user[item]['org_id']: logging.info("OrgManagement - try to add user \"" + list_user[item]['email'] + "\"") usr = MISPUser() usr.email = list_user[item]['email'] usr.org_id = org.id usr.role_id = list_user[item]['role_id'] usr_response = self._misp.add_user(usr, pythonify=True) # legnth of regular authkeys is 40 chars or longer self.assertTrue( usr_response.email in list_user[item]['email'], msg="The created users has no or a wrong email") self.assertTrue( usr_response.role_id in list_user[item]['role_id'], msg="The created users has no or a wrong role id") self.assertGreaterEqual( len(usr_response.authkey), 40, msg= "MISP responded with a wrong authkey - should be exactly 40 chars" ) ### An authentication test could be inserted here ### An user change role test could be inserted here logging.info( "OrgManagement - check if all user where created successfully") response = self._misp.users(pythonify=True) self.assertGreaterEqual( len(response), len(list_user), msg= "MISP responded with a wrong number of users - it seems that not all users could be created." ) for item in response: if item.org_id not in '1' or item.id not in '1': logging.info("OrgManagement - try to delete user \"" + item.email + "\"") usr_response = self._misp.delete_user(item) self.assertTrue("User deleted" in usr_response['message'], msg="User could ne be deleted") pass logging.info( "OrgManagement - check if user list now only contains the admin user" ) response = self._misp.users(pythonify=True) self.assertEqual( len(response), 1, "MISP responded with a wrong number of users - it seems that not all users could be deleted." ) ### Remove organisations response = self._misp.organisations(pythonify=True) for item in response: if item.id not in "1": logging.info("Try to remove organization: \"" + item.name + "\"") org_response = self._misp.delete_organisation(item) self.assertTrue( 'deleted' in org_response['message'], msg="Organisations could not be deleted from MISP") pass response = self._misp.organisations(pythonify=True) logging.info("OrgManagement - check if only admin org exist") self.assertEqual( len(response), 1, msg= "MISP responded with a wrong number of organisations - it seems that not all organisations could be deleted." )
# CSV format # orgname,nationality,sector,type,contacts,uuid,local,sharingroup with open(args.csv_import) as csv_file: count_orgs = 0 csv_reader = csv.reader(csv_file, delimiter=',') for row in csv_reader: org = MISPOrganisation() org.name = row[0] print("Process {}".format(org.name)) org.nationality = row[1] org.sector = row[2] org.type = row[3] org.contacts = row[4] org.uuid = row[5] org.local = row[6] add_org = misp.add_organisation(org, pythonify=True) if 'errors' in add_org: print(add_org['errors']) else: count_orgs = count_orgs + 1 org_uuid = add_org.uuid if org_uuid: sharinggroup = MISPSharingGroup() sharinggroup_uuid = row[7] if sharinggroup_uuid: