def create_daily_event(self):
today = str(datetime.date.today())
event_dict = {
'id': len(self.manifest) + 1,
'Tag': settings.Tag,
'info': self.daily_event_name.format(today),
'analysis': settings.analysis, # [0-2]
'threat_level_id': settings.threat_level_id, # [1-4]
'published': settings.published,
'date': today
}
event = MISPEvent()
event.from_dict(**event_dict)
# reference org
org = MISPOrganisation()
org.name = settings.org_name
org.uuid = settings.org_uuid
event.Orgc = org
# save event on disk
self.flush_event(new_event=event)
# add event to manifest
self.manifest.update(event.manifest)
self.save_manifest()
return event
def test_feed(self):
me = MISPEvent()
me.info = 'Test feed'
org = MISPOrganisation()
org.name = 'TestOrg'
org.uuid = '123478'
me.Orgc = org
me.add_attribute('ip-dst', '8.8.8.8')
obj = me.add_object(name='file')
obj.add_attributes('filename', *['foo.exe', 'bar.exe'])
h = hashlib.new('md5')
h.update(b'8.8.8.8')
hash_attr_val = h.hexdigest()
feed = me.to_feed(with_meta=True)
self.assertEqual(feed['Event']['_hashes'][0], hash_attr_val)
self.assertEqual(feed['Event']['_manifest'][me.uuid]['info'], 'Test feed')
self.assertEqual(len(feed['Event']['Object'][0]['Attribute']), 2)
def poll_taxii():
global f_hashes, f_manifest, f_events
results_dict = {}
client = create_client(
CYBERSAIYAN_FEED_URL,
use_https=TAXII_USE_TLS,
discovery_path=TAXII_DISCOVERY_PATH
)
blocks = client.poll(collection_name=CYBERSAIYAN_COLLECTION_NAME)
for block in blocks:
content = block.content
if content:
if type(content) == str:
continue
elif type(content) == bytes:
content = content.decode('utf-8')
pkg = STIXPackage.from_xml(StringIO(content))
title = pkg.stix_header.title
information_source = pkg.stix_header.information_source.identity.name
cs_event = (title, information_source)
cs_event_hash = hash(cs_event)
db_cursor.execute("SELECT uuid FROM hashes WHERE hash = '%s'" % cs_event_hash)
element = db_cursor.fetchone()
if element:
e_uuid = element[0]
else:
e_uuid = str(uuid.uuid4())
db_cursor.execute("INSERT INTO hashes VALUES (?,?)", (cs_event_hash,e_uuid,))
if cs_event_hash not in results_dict:
results_dict[cs_event_hash] = MISPEvent()
m_ev = results_dict[cs_event_hash]
m_ev.info = str(pkg.stix_header.description)
m_ev.analysis = 0
m_ev.uuid = e_uuid
#m_ev.org = "CyberSaiyan"
csorg = MISPOrganisation()
csorg.name = "CyberSaiyan"
csorg.uuid = "8aaa81ed-72ef-4fb1-8e96-fa1bc200faeb"
m_ev.orgc = csorg
marking = pkg.stix_header.handling.marking
tlp = 0
found_tlp = False
for m in marking:
for struct in m.marking_structures:
if struct._XSI_TYPE == "tlpMarking:TLPMarkingStructureType":
found_tlp = True
tlp = max(TLP[struct.color.lower()], tlp)
if tlp == 0 and not found_tlp:
tlp = TLP["amber"]
m_ev.add_tag("tlp:"+TLP[tlp])
m_ev.add_tag("CyberSaiyan")
indicators = pkg.indicators
last_ts = utc.localize(datetime.datetime(1970,1,1))
for indicator in indicators:
cur_ts = indicator.timestamp
if cur_ts > last_ts:
last_ts = cur_ts
obj = indicator.observable.object_
obj_d = obj.properties.to_dict()
attr_type = obj_d["xsi:type"]
if attr_type == "AddressObjectType":
attr = MISPAttribute()
attr.category = "Network activity"
attr.type = "ip-dst"
attr.value = obj_d["address_value"]
attr.disable_correlation = False
attr.to_ids = True
elif attr_type == "DomainNameObjectType":
attr = MISPAttribute()
attr.category = "Network activity"
attr.type = "domain"
attr.value = obj_d["value"]
attr.disable_correlation = False
attr.to_ids = True
elif attr_type == "URIObjectType":
attr = MISPAttribute()
attr.category = "Network activity"
attr.type = "url"
attr.value = obj_d["value"]
attr.disable_correlation = False
attr.to_ids = True
elif attr_type == "FileObjectType":
hash_type = obj_d["hashes"][0]["type"]["value"].lower()
hash_value = obj_d["hashes"][0]["simple_hash_value"]
attr = MISPAttribute()
attr.category = "Payload delivery"
assert hash_type in ('md5', "sha1", "sha224",
"sha256", "sha384", "sha512", "ssdeep")
attr.type = hash_type
attr.value = hash_value
attr.disable_correlation = False
attr.to_ids = True
m_ev.date = last_ts.strftime("%Y-%m-%d")
m_ev.attributes.append(attr)
db_conn.commit()
c_hashes, c_manifest, c_events = list(), dict(), dict()
for event in results_dict.values():
e_feed = event.to_feed(with_meta=True).get("Event")
c_hashes += [[h, event.uuid] for h in e_feed.pop("_hashes")]
c_manifest.update(e_feed.pop('_manifest'))
c_events[event.uuid] = e_feed
f_hashes, f_manifest, f_events = c_hashes, c_manifest, c_events
# initialize PyMISP and set url for Panorama
misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert)
urlVap = "https://tap-api-v2.proofpoint.com/v2/people/vap?window=30" # Window can be 14, 30, and 90 Days
headers = {'Authorization': "Basic " + proofpoint_key}
responseVap = requests.request("GET", urlVap, headers=headers)
jsonDataVap = json.loads(responseVap.text)
for alert in jsonDataVap["users"]:
orgc = MISPOrganisation()
orgc.name = 'Proofpoint'
orgc.id = '#{ORGC.ID}' # organisation id
orgc.uuid = '#{ORGC.UUID}' # organisation uuid
# initialize and set MISPEvent()
event = MISPEvent()
event.Orgc = orgc
event.info = 'Very Attacked Person ' + jsonDataVap["interval"]
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 2 # setting this to 0 breaks the integration
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
totalVapUsers = event.add_attribute('counter',
jsonDataVap["totalVapUsers"],
comment="Total VAP Users")
averageAttackIndex = event.add_attribute('counter',
jsonDataVap["averageAttackIndex"],
comment="Average Attack Count")
from pathlib import Path
from pymisp import MISPEvent, MISPOrganisation, PyMISP
from dateutil.parser import parse
import json
from pymisp.tools import feed_meta_generator
from io import BytesIO
make_feed = False
path = Path('/home/raphael/gits/covid-19-china/data')
if make_feed:
org = MISPOrganisation()
org.name = 'CIRCL'
org.uuid = "55f6ea5e-2c60-40e5-964f-47a8950d210f"
else:
from covid_key import url, key
misp = PyMISP(url, key)
for p in path.glob('*_json/current_china.json'):
d = parse(p.parent.name[:-5])
event = MISPEvent()
event.info = f"[{d.isoformat()}] DXY COVID-19 live report"
event.date = d
event.distribution = 3
event.add_tag('tlp:white')
if make_feed:
event.orgc = org
else:
e = misp.search(eventinfo=event.info, metadata=True, pythonify=True)
json_import = sys.argv[1]
event_import_org = sys.argv[2]
event_import_uuid = str(uuid.uuid4()) # Unique ID
event_import_date = date.today() # Create event with current data
event_import_distribution = 2 # Connected
# Check if organisation already exist
org = MISPOrganisation()
try:
org.id = api.get_organisation(event_import_org, pythonify=True).id
except:
# We need to create a new one
org_new = MISPOrganisation()
org_new.name = event_import_org
org_new.uuid = str(uuid.uuid4())
org_new.type = "CSIRT"
org_new.sector = "Government"
org.id = api.add_organisation(org_new, pythonify=True).id
# Create the MISP event by loading the JSON file
# This will not add the attributes, but does add the event tags and galaxies
# We also add a random UUID for uniqueness
event = MISPEvent()
event.load_file(json_import)
event.uuid = event_import_uuid
if not event_import_info:
event_import_info = event.info
event.info = event_import_info
event.date = event_import_date
event.distribution = event_import_distribution
def test_Organisations(self):
"""
This test creats new organisations from the org.json file and adds user from the user.json file to it.
After all organisations and user are created the tests removes them and checks if they are correctly removed
"""
### Create organisations from org.json file
org_list = readInFile("samples/org.json")
tested_keys = [
'name', 'description', 'nationality', 'sector', 'uuid', 'contacts'
]
for item in org_list:
org = MISPOrganisation()
org.name = org_list[item]['name']
org.description = org_list[item]['description']
org.nationality = org_list[item]['nationality']
org.sector = org_list[item]['sector']
org.uuid = org_list[item]['uuid']
org.contacts = org_list[item]['contacts']
#org.local = org_list[item]['local']
logging.info("OrgManagement - try to create organization \"" +
org_list[item]['name'] + "\"")
response = self._misp.add_organisation(org, pythonify=True)
self.assertTrue(
org_list[item]['uuid'] in response.uuid,
msg="The created organisation has no or a wrong UUID")
self.assertTrue(
org_list[item]['name'] in response.name,
msg="The created organisation has no or a wrong name")
self.assertTrue(
org_list[item]['description'] in response.description,
msg="The created organisation has no or a wrong description")
self.assertTrue(
org_list[item]['nationality'] in response.nationality,
msg="The created organisation has no or a wrong nationality")
self.assertTrue(
response.local,
msg=
"The created organisation is not a local organisation but should be a local organisation"
)
self.assertTrue(
org_list[item]['sector'] in response.sector,
msg="The created organisation has no or a wrong sector")
response = self._misp.organisations(scope="local", pythonify=True)
logging.info(
"OrgManagement - check if the admin and both test organisations exist"
)
self.assertGreaterEqual(
len(response), 3,
"MISP responded with less then 3 existing organisations - there shold exactly be 3"
)
### Add new user from the user.json file
list_user = readInFile("samples/user.json")
users = self._misp.users(pythonify=True)
for item in list_user:
for org in response:
if org.name in list_user[item]['org_id']:
logging.info("OrgManagement - try to add user \"" +
list_user[item]['email'] + "\"")
usr = MISPUser()
usr.email = list_user[item]['email']
usr.org_id = org.id
usr.role_id = list_user[item]['role_id']
usr_response = self._misp.add_user(usr, pythonify=True)
# legnth of regular authkeys is 40 chars or longer
self.assertTrue(
usr_response.email in list_user[item]['email'],
msg="The created users has no or a wrong email")
self.assertTrue(
usr_response.role_id in list_user[item]['role_id'],
msg="The created users has no or a wrong role id")
self.assertGreaterEqual(
len(usr_response.authkey),
40,
msg=
"MISP responded with a wrong authkey - should be exactly 40 chars"
)
### An authentication test could be inserted here
### An user change role test could be inserted here
logging.info(
"OrgManagement - check if all user where created successfully")
response = self._misp.users(pythonify=True)
self.assertGreaterEqual(
len(response),
len(list_user),
msg=
"MISP responded with a wrong number of users - it seems that not all users could be created."
)
for item in response:
if item.org_id not in '1' or item.id not in '1':
logging.info("OrgManagement - try to delete user \"" +
item.email + "\"")
usr_response = self._misp.delete_user(item)
self.assertTrue("User deleted" in usr_response['message'],
msg="User could ne be deleted")
pass
logging.info(
"OrgManagement - check if user list now only contains the admin user"
)
response = self._misp.users(pythonify=True)
self.assertEqual(
len(response), 1,
"MISP responded with a wrong number of users - it seems that not all users could be deleted."
)
### Remove organisations
response = self._misp.organisations(pythonify=True)
for item in response:
if item.id not in "1":
logging.info("Try to remove organization: \"" + item.name +
"\"")
org_response = self._misp.delete_organisation(item)
self.assertTrue(
'deleted' in org_response['message'],
msg="Organisations could not be deleted from MISP")
pass
response = self._misp.organisations(pythonify=True)
logging.info("OrgManagement - check if only admin org exist")
self.assertEqual(
len(response),
1,
msg=
"MISP responded with a wrong number of organisations - it seems that not all organisations could be deleted."
)
# CSV format
# orgname,nationality,sector,type,contacts,uuid,local,sharingroup
with open(args.csv_import) as csv_file:
count_orgs = 0
csv_reader = csv.reader(csv_file, delimiter=',')
for row in csv_reader:
org = MISPOrganisation()
org.name = row[0]
print("Process {}".format(org.name))
org.nationality = row[1]
org.sector = row[2]
org.type = row[3]
org.contacts = row[4]
org.uuid = row[5]
org.local = row[6]
add_org = misp.add_organisation(org, pythonify=True)
if 'errors' in add_org:
print(add_org['errors'])
else:
count_orgs = count_orgs + 1
org_uuid = add_org.uuid
if org_uuid:
sharinggroup = MISPSharingGroup()
sharinggroup_uuid = row[7]
if sharinggroup_uuid:
