def create_daily_event(self):
today = str(datetime.date.today())
event_dict = {
'id': len(self.manifest) + 1,
'Tag': settings.Tag,
'info': self.daily_event_name.format(today),
'analysis': settings.analysis, # [0-2]
'threat_level_id': settings.threat_level_id, # [1-4]
'published': settings.published,
'date': today
}
event = MISPEvent()
event.from_dict(**event_dict)
# reference org
org = MISPOrganisation()
org.name = settings.org_name
org.uuid = settings.org_uuid
event.Orgc = org
# save event on disk
self.flush_event(new_event=event)
# add event to manifest
self.manifest.update(event.manifest)
self.save_manifest()
return event
def generate_MISP_Event(deduplicated_observations, conf, tags, attr_tags):
dt = datetime.now()
event = MISPEvent()
event.info = dt.strftime("%Y%m%d ") + 'TIE'
event.publish_timestamp = dt.strftime("%s")
event.timestamp = dt.strftime("%s")
event['timestamp'] = dt.strftime("%s")
event.analysis = 2
event.published = conf.event_published
orgc = MISPOrganisation()
orgc.from_json(json.dumps({'name': conf.org_name, 'uuid': conf.org_uuid}))
event.orgc = orgc
event.threat_level_id = conf.event_base_thread_level
event.date = dt
event['uuid'] = str(uuid.uuid1())
if len(tags) > 0:
event['Tag'] = tags
attr_hashes = []
for key, attr in deduplicated_observations.items():
misp_attr = MISPAttribute()
misp_attr.timestamp = dt.strftime("%s")
misp_attr['timestamp'] = dt.strftime("%s")
misp_attr.type = get_Attribute_Type(attr)
misp_attr.value = get_MISP_Fitted_Value(attr["value"], misp_attr.type)
if 'c2-server' in attr['categories'] and attr_tags.c2tags:
misp_attr['Tag'] = attr_tags.c2tags
if 'malware' in attr['categories'] and attr_tags.malwaretags:
misp_attr['Tag'] = attr_tags.malwaretags
if 'espionage' in attr['categories'] and attr_tags.espionagetags:
misp_attr['Tag'] = attr_tags.espionagetags
if 'bot' in attr['categories'] and attr_tags.bottags:
misp_attr['Tag'] = attr_tags.bottags
if 'whitelist' in attr['categories'] and attr_tags.whitelisttags:
misp_attr['Tag'] = attr_tags.whitelisttags
if 'cybercrime' in attr['categories'] and attr_tags.cybercrimetags:
misp_attr['Tag'] = attr_tags.cybercrimetags
if 'phishing' in attr['categories'] and attr_tags.phishingtags:
misp_attr['Tag'] = attr_tags.phishingtags
misp_attr.category = get_Attribute_Category(attr)
if conf.attr_to_ids and attr[
'min_confidence'] >= conf.attr_to_ids_threshold:
misp_attr.to_ids = True
else:
misp_attr.to_ids = False
misp_attr['comment'] = 'categories: ' + str(attr['categories']) + ' actors: ' + str(attr['actors']) + \
' families: ' + str(attr['families']) + ' sources: ' + str(attr['sources']) + \
' severity: ' + str(attr['max_severity']) + \
' confidence: ' + str(attr['max_confidence'])
misp_attr.edited = False
event.add_attribute(**(misp_attr.to_dict()))
attr_hashes.append([
hashlib.md5(attr['value'].encode("utf-8")).hexdigest(),
event['uuid']
])
event.edited = False
return event, attr_hashes
def __init__(self, params):
self.initial_user_connector = ExpandedPyMISP(params['url'], params['key'], ssl=False, debug=False)
# Git pull
self.initial_user_connector.update_misp()
# Set the default role (id 3 on the VM is normal user)
self.initial_user_connector.set_default_role(3)
# Restart workers
self.initial_user_connector.restart_workers()
if not fast_mode:
# Load submodules
self.initial_user_connector.update_object_templates()
self.initial_user_connector.update_galaxies()
self.initial_user_connector.update_noticelists()
self.initial_user_connector.update_warninglists()
self.initial_user_connector.update_taxonomies()
self.initial_user_connector.toggle_global_pythonify()
# Create organisation
organisation = MISPOrganisation()
organisation.name = params['orgname']
self.test_org = self.initial_user_connector.add_organisation(organisation)
print(self.test_org.name, self.test_org.uuid)
# Create Site admin in new org
user = MISPUser()
user.email = params['email_site_admin']
user.org_id = self.test_org.id
user.role_id = 1 # Site admin
self.test_site_admin = self.initial_user_connector.add_user(user)
self.site_admin_connector = ExpandedPyMISP(params['url'], self.test_site_admin.authkey, ssl=False, debug=False)
self.site_admin_connector.toggle_global_pythonify()
# Create org admin
user = MISPUser()
user.email = params['email_admin']
user.org_id = self.test_org.id
user.role_id = 2 # Org admin
self.test_org_admin = self.site_admin_connector.add_user(user)
self.org_admin_connector = ExpandedPyMISP(params['url'], self.test_org_admin.authkey, ssl=False, debug=False)
self.org_admin_connector.toggle_global_pythonify()
# Create user
user = MISPUser()
user.email = params['email_user']
user.org_id = self.test_org.id
self.test_usr = self.org_admin_connector.add_user(user)
self.user_connector = ExpandedPyMISP(params['url'], self.test_usr.authkey, ssl=False, debug=False)
self.user_connector.toggle_global_pythonify()
# Setup external_baseurl
self.site_admin_connector.set_server_setting('MISP.external_baseurl', params['external_baseurl'], force=True)
# Setup baseurl
self.site_admin_connector.set_server_setting('MISP.baseurl', params['url'], force=True)
# Setup host org
self.site_admin_connector.set_server_setting('MISP.host_org_id', self.test_org.id)
self.external_base_url = params['external_baseurl']
self.sync = []
self.sync_servers = []
def setUpClass(cls):
warnings.simplefilter("ignore", ResourceWarning)
# Connect as admin
cls.admin_misp_connector = PyMISP(url, key)
cls.admin_misp_connector.set_server_setting('debug', 1, force=True)
cls.admin_misp_connector.global_pythonify = True
# Check if admin is really site admin
assert cls.admin_misp_connector._current_role.perm_site_admin
# Create advanced authkey, so connector will work even after advanced keys are required
cls.admin_advanced_authkey = cls.__create_advanced_authkey(
cls, cls.admin_misp_connector._current_user.id)
cls.admin_misp_connector.key = cls.admin_misp_connector.key + "," + cls.admin_advanced_authkey[
"authkey_raw"]
# Creates an org
organisation = MISPOrganisation()
organisation.name = 'Test Org ' + random() # make name always unique
cls.test_org = cls.admin_misp_connector.add_organisation(organisation)
check_response(cls.test_org)
# Creates org admin
org_admin = MISPUser()
org_admin.email = 'testorgadmin@user' + random(
) + '.local' # make name always unique
org_admin.org_id = cls.test_org.id
org_admin.role_id = 2 # Org admin role
cls.test_org_admin = cls.admin_misp_connector.add_user(org_admin)
check_response(cls.test_org_admin)
# Creates advanced auth key for org admin
cls.org_admin_advanced_authkey = cls.__create_advanced_authkey(
cls, cls.test_org_admin.id)
cls.org_admin_misp_connector = PyMISP(
url, cls.test_org_admin.authkey + "," +
cls.org_admin_advanced_authkey["authkey_raw"])
cls.org_admin_misp_connector.global_pythonify = True
# Creates an user
cls.test_usr_password = str(uuid.uuid4())
user = MISPUser()
user.email = 'testusr@user' + random(
) + '.local' # make name always unique
user.org_id = cls.test_org.id
user.role_id = 3 # User role
user.password = cls.test_usr_password
cls.test_usr = cls.admin_misp_connector.add_user(user)
check_response(cls.test_usr)
# Try to connect as user to check if everything works
PyMISP(url, cls.test_usr.authkey)
# Check if user can login with given password
assert login(url, cls.test_usr.email, cls.test_usr_password)
def setUpClass(cls):
cls.maxDiff = None
# Connect as admin
cls.admin_misp_connector = PyMISP(url, key, ssl=False, debug=False)
cls.admin_misp_connector.set_server_setting('debug', 1, force=True)
# Creates an org
organisation = MISPOrganisation()
organisation.name = 'Test Org'
cls.test_org = cls.admin_misp_connector.add_organisation(organisation, pythonify=True)
# Set the default role (id 3 on the VM)
cls.admin_misp_connector.set_default_role(3)
# Creates a user
user = MISPUser()
user.email = '*****@*****.**'
user.org_id = cls.test_org.id
cls.test_usr = cls.admin_misp_connector.add_user(user, pythonify=True)
cls.user_misp_connector = PyMISP(url, cls.test_usr.authkey, ssl=False, debug=True)
def test_feed(self):
me = MISPEvent()
me.info = 'Test feed'
org = MISPOrganisation()
org.name = 'TestOrg'
org.uuid = '123478'
me.Orgc = org
me.add_attribute('ip-dst', '8.8.8.8')
obj = me.add_object(name='file')
obj.add_attributes('filename', *['foo.exe', 'bar.exe'])
h = hashlib.new('md5')
h.update(b'8.8.8.8')
hash_attr_val = h.hexdigest()
feed = me.to_feed(with_meta=True)
self.assertEqual(feed['Event']['_hashes'][0], hash_attr_val)
self.assertEqual(feed['Event']['_manifest'][me.uuid]['info'], 'Test feed')
self.assertEqual(len(feed['Event']['Object'][0]['Attribute']), 2)
def __init__(self, config, logger):
self.logger = logger
MISP_KEY = config['MISP_KEY']
MISP_URL = config['MISP_URL']
MISP_VERIFYCERT = config['MISP_VERIFYCERT']
self.config = config
self.misp = ExpandedPyMISP(MISP_URL, MISP_KEY, MISP_VERIFYCERT)
self.orgc = MISPOrganisation()
self.orgc.name = config['MISP_ORG_NAME']
self.orgc.id = config['MISP_ORG_ID']
self.orgc.uuid = config['MISP_ORG_UUID']
self.tags = config['tags']
self.galaxy_synonyms = {}
self.enabled_clusters = self.config['galaxies']
self.galaxy_tags = {}
self._init_galaxies()
def setUpClass(cls):
cls.maxDiff = None
# Connect as admin
cls.admin_misp_connector = ExpandedPyMISP(url, key)
# Creates an org
org = cls.admin_misp_connector.add_organisation(name='Test Org')
cls.test_org = MISPOrganisation()
cls.test_org.from_dict(**org)
# Creates a user
usr = cls.admin_misp_connector.add_user(email='*****@*****.**', org_id=cls.test_org.id, role_id=3)
cls.test_usr = MISPUser()
cls.test_usr.from_dict(**usr)
cls.user_misp_connector = ExpandedPyMISP(url, cls.test_usr.authkey)
# Creates a publisher
pub = cls.admin_misp_connector.add_user(email='*****@*****.**', org_id=cls.test_org.id, role_id=4)
cls.test_pub = MISPUser()
cls.test_pub.from_dict(**pub)
cls.pub_misp_connector = ExpandedPyMISP(url, cls.test_pub.authkey)
def init(self):
if MISPEvent is None and import_fail_reason == 'syntax':
raise MissingDependencyError("pymisp",
version='>=2.4.117.3',
additional_text="Python versions below 3.6 are "
"only supported by pymisp <= 2.4.119.1.")
elif MISPEvent is None:
raise MissingDependencyError('pymisp', version='>=2.4.117.3')
self.current_event = None
self.misp_org = MISPOrganisation()
self.misp_org.name = self.misp_org_name
self.misp_org.uuid = self.misp_org_uuid
self.output_dir = Path(self.output_dir)
MISPFeedOutputBot.check_output_dir(self.output_dir)
if self.interval_event is None:
self.timedelta = datetime.timedelta(hours=1)
else:
self.timedelta = datetime.timedelta(minutes=parse_relative(self.interval_event))
if (self.output_dir / '.current').exists():
with (self.output_dir / '.current').open() as f:
self.current_file = Path(f.read())
self.current_event = MISPEvent()
self.current_event.load_file(self.current_file)
last_min_time, last_max_time = re.findall('IntelMQ event (.*) - (.*)', self.current_event.info)[0]
last_min_time = datetime.datetime.strptime(last_min_time, '%Y-%m-%dT%H:%M:%S.%f')
last_max_time = datetime.datetime.strptime(last_max_time, '%Y-%m-%dT%H:%M:%S.%f')
if last_max_time < datetime.datetime.now():
self.min_time_current = datetime.datetime.now()
self.max_time_current = self.min_time_current + self.timedelta
self.current_event = None
else:
self.min_time_current = last_min_time
self.max_time_current = last_max_time
else:
self.min_time_current = datetime.datetime.now()
self.max_time_current = self.min_time_current + self.timedelta
def __init__(self, output_dir: str= 'output', org_name: str='CIRCL',
org_uuid: str='55f6ea5e-2c60-40e5-964f-47a8950d210f'):
self.misp_org = MISPOrganisation()
self.misp_org.name = org_name
self.misp_org.uuid = org_uuid
self.output_dir = Path(output_dir)
self.output_dir.mkdir(exist_ok=True)
self.data_dir = self.output_dir / 'data'
self.data_dir.mkdir(exist_ok=True)
self.scrippts_meta_file = self.output_dir / '.meta_scrippts'
self.scrippts_meta = {}
if self.scrippts_meta_file.exists():
# Format: <infofield>,<uuid>.json
with self.scrippts_meta_file.open() as f:
reader = csv.reader(f)
for row in reader:
self.scrippts_meta[row[0]] = row[1]
else:
self.scrippts_meta_file.touch()
def init(self):
if MISPEvent is None:
raise MissingDependencyError('pymisp', version='>=2.4.117.3')
self.current_event = None
self.misp_org = MISPOrganisation()
self.misp_org.name = self.parameters.misp_org_name
self.misp_org.uuid = self.parameters.misp_org_uuid
self.output_dir = Path(self.parameters.output_dir)
MISPFeedOutputBot.check_output_dir(self.output_dir)
if not hasattr(self.parameters, 'interval_event'):
self.timedelta = datetime.timedelta(hours=1)
else:
self.timedelta = datetime.timedelta(minutes=parse_relative(self.parameters.interval_event))
if (self.output_dir / '.current').exists():
with (self.output_dir / '.current').open() as f:
self.current_file = Path(f.read())
self.current_event = MISPEvent()
self.current_event.load_file(self.current_file)
last_min_time, last_max_time = re.findall('IntelMQ event (.*) - (.*)', self.current_event.info)[0]
last_min_time = datetime.datetime.strptime(last_min_time, '%Y-%m-%dT%H:%M:%S.%f')
last_max_time = datetime.datetime.strptime(last_max_time, '%Y-%m-%dT%H:%M:%S.%f')
if last_max_time < datetime.datetime.now():
self.min_time_current = datetime.datetime.now()
self.max_time_current = self.min_time_current + self.timedelta
self.current_event = None
else:
self.min_time_current = last_min_time
self.max_time_current = last_max_time
else:
self.min_time_current = datetime.datetime.now()
self.max_time_current = self.min_time_current + self.timedelta
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Add organizations from a CSV file')
parser.add_argument("-c", "--csv-import", required=True, help="The CSV file containing the organizations. Format 'orgname,nationality,sector,type,contacts,uuid,local,sharingroup_uuid'")
args = parser.parse_args()
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
# CSV format
# orgname,nationality,sector,type,contacts,uuid,local,sharingroup
with open(args.csv_import) as csv_file:
count_orgs = 0
csv_reader = csv.reader(csv_file, delimiter=',')
for row in csv_reader:
org = MISPOrganisation()
org.name = row[0]
print("Process {}".format(org.name))
org.nationality = row[1]
org.sector = row[2]
org.type = row[3]
org.contacts = row[4]
org.uuid = row[5]
org.local = row[6]
add_org = misp.add_organisation(org, pythonify=True)
if 'errors' in add_org:
print(add_org['errors'])
else:
count_orgs = count_orgs + 1
def __init__(self, misp_instance_dir: Path, secure_connection: bool):
with (misp_instance_dir / 'config.json').open() as f:
self.instance_config = json.load(f)
print('Initialize', self.instance_config['admin_orgname'])
self.secure_connection = secure_connection
self.synchronisations = {}
self.name = self.instance_config['admin_orgname']
# NOTE: never use that user again after initial config.
initial_user_connector = PyMISP(self.instance_config['baseurl'],
self.instance_config['admin_key'],
ssl=self.secure_connection,
debug=False)
# Set the default role (id 3 is normal user)
initial_user_connector.set_default_role(3)
initial_user_connector.toggle_global_pythonify()
self.baseurl = self.instance_config['baseurl']
self.external_baseurl = self.instance_config['external_baseurl']
# Create organisation
organisation = MISPOrganisation()
organisation.name = self.instance_config['admin_orgname']
self.host_org = initial_user_connector.add_organisation(organisation)
if not isinstance(self.host_org, MISPOrganisation):
# The organisation is probably already there
organisations = initial_user_connector.organisations()
for organisation in organisations:
if organisation.name == self.instance_config['admin_orgname']:
self.host_org = organisation
break
else:
raise Exception('Unable to find admin organisation')
# Create Site admin in new org
user = MISPUser()
user.email = self.instance_config['email_site_admin']
user.org_id = self.host_org.id
user.role_id = 1 # Site admin
self.host_site_admin = initial_user_connector.add_user(user)
if not isinstance(self.host_site_admin, MISPUser):
users = initial_user_connector.users()
for user in users:
if user.email == self.instance_config['email_site_admin']:
self.host_site_admin = user
break
else:
raise Exception('Unable to find admin user')
self.site_admin_connector = PyMISP(self.baseurl,
self.host_site_admin.authkey,
ssl=self.secure_connection,
debug=False)
self.site_admin_connector.toggle_global_pythonify()
# Setup external_baseurl
self.site_admin_connector.set_server_setting('MISP.external_baseurl',
self.external_baseurl,
force=True)
# Setup baseurl
self.site_admin_connector.set_server_setting('MISP.baseurl',
self.baseurl,
force=True)
# Setup host org
self.site_admin_connector.set_server_setting('MISP.host_org_id',
self.host_org.id)
# create other useful users
self.orgadmin = self.create_user(
self.instance_config['email_orgadmin'], 2)
self.user = self.create_user(self.instance_config['email_user'], 3)
# And connectors
self.org_admin_connector = PyMISP(self.baseurl,
self.orgadmin.authkey,
ssl=self.secure_connection,
debug=False)
self.org_admin_connector.toggle_global_pythonify()
self.user_connector = PyMISP(self.baseurl,
self.user.authkey,
ssl=self.secure_connection,
debug=False)
self.user_connector.toggle_global_pythonify()
from pymisp import ExpandedPyMISP, MISPEvent, MISPOrganisation
from keys import misp_url, misp_key, misp_verifycert, proofpoint_key
# initialize PyMISP and set url for Panorama
misp = ExpandedPyMISP(url=misp_url, key=misp_key, ssl=misp_verifycert)
urlVap = "https://tap-api-v2.proofpoint.com/v2/people/vap?window=30" # Window can be 14, 30, and 90 Days
headers = {'Authorization': "Basic " + proofpoint_key}
responseVap = requests.request("GET", urlVap, headers=headers)
jsonDataVap = json.loads(responseVap.text)
for alert in jsonDataVap["users"]:
orgc = MISPOrganisation()
orgc.name = 'Proofpoint'
orgc.id = '#{ORGC.ID}' # organisation id
orgc.uuid = '#{ORGC.UUID}' # organisation uuid
# initialize and set MISPEvent()
event = MISPEvent()
event.Orgc = orgc
event.info = 'Very Attacked Person ' + jsonDataVap["interval"]
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 2 # setting this to 0 breaks the integration
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
totalVapUsers = event.add_attribute('counter',
jsonDataVap["totalVapUsers"],
comment="Total VAP Users")
def poll_taxii():
global f_hashes, f_manifest, f_events
results_dict = {}
client = create_client(
CYBERSAIYAN_FEED_URL,
use_https=TAXII_USE_TLS,
discovery_path=TAXII_DISCOVERY_PATH
)
blocks = client.poll(collection_name=CYBERSAIYAN_COLLECTION_NAME)
for block in blocks:
content = block.content
if content:
if type(content) == str:
continue
elif type(content) == bytes:
content = content.decode('utf-8')
pkg = STIXPackage.from_xml(StringIO(content))
title = pkg.stix_header.title
information_source = pkg.stix_header.information_source.identity.name
cs_event = (title, information_source)
cs_event_hash = hash(cs_event)
db_cursor.execute("SELECT uuid FROM hashes WHERE hash = '%s'" % cs_event_hash)
element = db_cursor.fetchone()
if element:
e_uuid = element[0]
else:
e_uuid = str(uuid.uuid4())
db_cursor.execute("INSERT INTO hashes VALUES (?,?)", (cs_event_hash,e_uuid,))
if cs_event_hash not in results_dict:
results_dict[cs_event_hash] = MISPEvent()
m_ev = results_dict[cs_event_hash]
m_ev.info = str(pkg.stix_header.description)
m_ev.analysis = 0
m_ev.uuid = e_uuid
#m_ev.org = "CyberSaiyan"
csorg = MISPOrganisation()
csorg.name = "CyberSaiyan"
csorg.uuid = "8aaa81ed-72ef-4fb1-8e96-fa1bc200faeb"
m_ev.orgc = csorg
marking = pkg.stix_header.handling.marking
tlp = 0
found_tlp = False
for m in marking:
for struct in m.marking_structures:
if struct._XSI_TYPE == "tlpMarking:TLPMarkingStructureType":
found_tlp = True
tlp = max(TLP[struct.color.lower()], tlp)
if tlp == 0 and not found_tlp:
tlp = TLP["amber"]
m_ev.add_tag("tlp:"+TLP[tlp])
m_ev.add_tag("CyberSaiyan")
indicators = pkg.indicators
last_ts = utc.localize(datetime.datetime(1970,1,1))
for indicator in indicators:
cur_ts = indicator.timestamp
if cur_ts > last_ts:
last_ts = cur_ts
obj = indicator.observable.object_
obj_d = obj.properties.to_dict()
attr_type = obj_d["xsi:type"]
if attr_type == "AddressObjectType":
attr = MISPAttribute()
attr.category = "Network activity"
attr.type = "ip-dst"
attr.value = obj_d["address_value"]
attr.disable_correlation = False
attr.to_ids = True
elif attr_type == "DomainNameObjectType":
attr = MISPAttribute()
attr.category = "Network activity"
attr.type = "domain"
attr.value = obj_d["value"]
attr.disable_correlation = False
attr.to_ids = True
elif attr_type == "URIObjectType":
attr = MISPAttribute()
attr.category = "Network activity"
attr.type = "url"
attr.value = obj_d["value"]
attr.disable_correlation = False
attr.to_ids = True
elif attr_type == "FileObjectType":
hash_type = obj_d["hashes"][0]["type"]["value"].lower()
hash_value = obj_d["hashes"][0]["simple_hash_value"]
attr = MISPAttribute()
attr.category = "Payload delivery"
assert hash_type in ('md5', "sha1", "sha224",
"sha256", "sha384", "sha512", "ssdeep")
attr.type = hash_type
attr.value = hash_value
attr.disable_correlation = False
attr.to_ids = True
m_ev.date = last_ts.strftime("%Y-%m-%d")
m_ev.attributes.append(attr)
db_conn.commit()
c_hashes, c_manifest, c_events = list(), dict(), dict()
for event in results_dict.values():
e_feed = event.to_feed(with_meta=True).get("Event")
c_hashes += [[h, event.uuid] for h in e_feed.pop("_hashes")]
c_manifest.update(e_feed.pop('_manifest'))
c_events[event.uuid] = e_feed
f_hashes, f_manifest, f_events = c_hashes, c_manifest, c_events
def test_Organisations(self):
"""
This test creats new organisations from the org.json file and adds user from the user.json file to it.
After all organisations and user are created the tests removes them and checks if they are correctly removed
"""
### Create organisations from org.json file
org_list = readInFile("samples/org.json")
tested_keys = [
'name', 'description', 'nationality', 'sector', 'uuid', 'contacts'
]
for item in org_list:
org = MISPOrganisation()
org.name = org_list[item]['name']
org.description = org_list[item]['description']
org.nationality = org_list[item]['nationality']
org.sector = org_list[item]['sector']
org.uuid = org_list[item]['uuid']
org.contacts = org_list[item]['contacts']
#org.local = org_list[item]['local']
logging.info("OrgManagement - try to create organization \"" +
org_list[item]['name'] + "\"")
response = self._misp.add_organisation(org, pythonify=True)
self.assertTrue(
org_list[item]['uuid'] in response.uuid,
msg="The created organisation has no or a wrong UUID")
self.assertTrue(
org_list[item]['name'] in response.name,
msg="The created organisation has no or a wrong name")
self.assertTrue(
org_list[item]['description'] in response.description,
msg="The created organisation has no or a wrong description")
self.assertTrue(
org_list[item]['nationality'] in response.nationality,
msg="The created organisation has no or a wrong nationality")
self.assertTrue(
response.local,
msg=
"The created organisation is not a local organisation but should be a local organisation"
)
self.assertTrue(
org_list[item]['sector'] in response.sector,
msg="The created organisation has no or a wrong sector")
response = self._misp.organisations(scope="local", pythonify=True)
logging.info(
"OrgManagement - check if the admin and both test organisations exist"
)
self.assertGreaterEqual(
len(response), 3,
"MISP responded with less then 3 existing organisations - there shold exactly be 3"
)
### Add new user from the user.json file
list_user = readInFile("samples/user.json")
users = self._misp.users(pythonify=True)
for item in list_user:
for org in response:
if org.name in list_user[item]['org_id']:
logging.info("OrgManagement - try to add user \"" +
list_user[item]['email'] + "\"")
usr = MISPUser()
usr.email = list_user[item]['email']
usr.org_id = org.id
usr.role_id = list_user[item]['role_id']
usr_response = self._misp.add_user(usr, pythonify=True)
# legnth of regular authkeys is 40 chars or longer
self.assertTrue(
usr_response.email in list_user[item]['email'],
msg="The created users has no or a wrong email")
self.assertTrue(
usr_response.role_id in list_user[item]['role_id'],
msg="The created users has no or a wrong role id")
self.assertGreaterEqual(
len(usr_response.authkey),
40,
msg=
"MISP responded with a wrong authkey - should be exactly 40 chars"
)
### An authentication test could be inserted here
### An user change role test could be inserted here
logging.info(
"OrgManagement - check if all user where created successfully")
response = self._misp.users(pythonify=True)
self.assertGreaterEqual(
len(response),
len(list_user),
msg=
"MISP responded with a wrong number of users - it seems that not all users could be created."
)
for item in response:
if item.org_id not in '1' or item.id not in '1':
logging.info("OrgManagement - try to delete user \"" +
item.email + "\"")
usr_response = self._misp.delete_user(item)
self.assertTrue("User deleted" in usr_response['message'],
msg="User could ne be deleted")
pass
logging.info(
"OrgManagement - check if user list now only contains the admin user"
)
response = self._misp.users(pythonify=True)
self.assertEqual(
len(response), 1,
"MISP responded with a wrong number of users - it seems that not all users could be deleted."
)
### Remove organisations
response = self._misp.organisations(pythonify=True)
for item in response:
if item.id not in "1":
logging.info("Try to remove organization: \"" + item.name +
"\"")
org_response = self._misp.delete_organisation(item)
self.assertTrue(
'deleted' in org_response['message'],
msg="Organisations could not be deleted from MISP")
pass
response = self._misp.organisations(pythonify=True)
logging.info("OrgManagement - check if only admin org exist")
self.assertEqual(
len(response),
1,
msg=
"MISP responded with a wrong number of organisations - it seems that not all organisations could be deleted."
)
"type":"report",
"async":"yes",
"reporttype":"predefined",
"reportname":"top-attacks",
"key":"#{API_KEY}" # put your Panorama api key here
}
# try with "..., verify=False)" if you get an SSL error
response = requests.request("GET", url, params=querystring)
resp_text = response.text
json_data = json.loads(json.dumps(xmltodict.parse(resp_text)))
# initialize and set MISPOrganisation
orgc = MISPOrganisation()
orgc.name = 'Palo Alto'
orgc.id = '#{ORGC_ID}' # organisation id
orgc.uuid = '#{ORGC_UUID}' # organisation uuid
# initialize and set MISPEvent()
event = MISPEvent()
event.Orgc = orgc
event.info = json_data['report']['result']['@name'] + " | " + json_data['report']['result']['@range']
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 0 # Optional, defaults to MISP.default_event_threat_level in MISP config
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
event.add_tag('firewall threats')
for threatid in json_data['report']['result']['entry']:
attribute = event.add_attribute('comment', threatid['threatid'])
attribute.comment = threatid['count']
def test_generate_MISP_event(self):
conf = Config.parse("settings/config.yml")
test_obs = {
"3ad54db13a7b6129902b0ee0acf3e2d1": {
"data_type": "IPv4",
"first_seen": "2019-08-21 08:38:29+02:00",
"last_seen": "2019-08-21 13:38:28+02:00",
"created_at": "2019-08-21 08:51:20.242089+02:00",
"updated_at": "2019-08-21 13:51:10.270419+02:00",
"max_confidence": 40,
"min_confidence": 40,
"max_severity": 1,
"min_severity": 1,
"n_occurrences": 1,
"sources": [{
"pseudonym": "testpseudo1",
"name": "testname1"
}],
"value": "123.45.67.89/32",
"categories": [],
"actors": [],
"families": []
},
"a3475b4484bed2a863720110e8099208": {
"data_type": "ExactHash",
"first_seen": "2019-08-21 13:38:17+02:00",
"last_seen": "2019-08-21 13:38:26+02:00",
"created_at": "2019-08-21 13:40:02.575150+02:00",
"updated_at": "2019-08-21 13:40:02.575150+02:00",
"max_confidence": 90,
"min_confidence": 90,
"max_severity": 1,
"min_severity": 1,
"n_occurrences": 1,
"sources": [{
"pseudonym": "testpseudo2",
"name": "testname2"
}],
"value": "sha1:930A0029225AA4C28B8EF095B679285EAAE27078",
"categories": [],
"actors": [],
"families": ["testfamily"]
}
}
event, attr_hashes = MISPHelper.generate_MISP_Event(test_obs, conf, [])
dt = datetime.now()
assert isinstance(event, MISPEvent)
assert event.info == dt.strftime("%Y%m%d ") + 'TIE'
assert event.publish_timestamp == dt.strftime("%s")
assert event.timestamp == dt.strftime("%s")
assert event['timestamp'] == dt.strftime("%s")
assert event.analysis == 2
assert event.published
orgc = MISPOrganisation()
orgc.from_json(
json.dumps({
'name': conf.org_name,
'uuid': conf.org_uuid
}))
assert event.orgc == orgc
assert event.threat_level_id == conf.event_base_thread_level
assert len(event['Attribute']) == 2
def host_org(self) -> MISPOrganisation:
organisation = MISPOrganisation()
organisation.name = self.config['admin_orgname']
return self.create_or_update_organisation(organisation)
from_time = to_time - timedelta(hours=6)
# convert to millis since epoch
to_time = datetime_to_millis(to_time)
from_time = datetime_to_millis(from_time)
rhisac = "7a33144f-aef3-442b-87d4-dbf70d8afdb0" # TruStar RH-ISAC enclave uuid
reports = tru.get_reports(from_time=from_time,
to_time=to_time,
is_enclave=True,
enclave_ids=rhisac)
# loop through each trustar report and create MISP event
for report in reports:
# initialize and set MISPOrganisation()
orgc = MISPOrganisation()
orgc.name = 'RH-ISAC'
orgc.id = '#{ORGC_ID}' # organisation id
orgc.uuid = '#{ORGC_UUID}' # organisation uuid
# initialize and set MISPEvent()
event = MISPEvent()
event.Orgc = orgc
event.info = report.title
event.distribution = 0 # Optional, defaults to MISP.default_event_distribution in MISP config
event.threat_level_id = 2 # Optional, defaults to MISP.default_event_threat_level in MISP config
event.analysis = 0 # Optional, defaults to 0 (initial analysis)
# get tags for report
for tag in tru.get_enclave_tags(report.id):
event.add_tag(tag.name)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from pathlib import Path
from pymisp import MISPEvent, MISPOrganisation, PyMISP
from dateutil.parser import parse
import json
from pymisp.tools import feed_meta_generator
from io import BytesIO
make_feed = False
path = Path('/home/raphael/gits/covid-19-china/data')
if make_feed:
org = MISPOrganisation()
org.name = 'CIRCL'
org.uuid = "55f6ea5e-2c60-40e5-964f-47a8950d210f"
else:
from covid_key import url, key
misp = PyMISP(url, key)
for p in path.glob('*_json/current_china.json'):
d = parse(p.parent.name[:-5])
event = MISPEvent()
event.info = f"[{d.isoformat()}] DXY COVID-19 live report"
event.date = d
event.distribution = 3
event.add_tag('tlp:white')
if make_feed:
event.orgc = org
event.from_dict(**loadEvent(result_uuid))
print("Checking for updated values in Event: " + event.uuid + " " +
event.info)
if event.info != EVENT_NAME: # information on event info is the same?
print("Event name changed, updading... old:" + event.info +
" new:" + EVENT_NAME)
event.info = EVENT_NAME
event_changed = 1
event.timestamp = now
else: # event does not exist, generate a new one
event = MISPEvent()
event.info = EVENT_NAME
event.analysis = 0 # events are created with Threat level Undefined and Analysis Initial
event.threat_level_id = 4
event.published = 0 # events are created unpublished
event.orgc = MISPOrganisation()
event.orgc.uuid = org_uuid
event.orgc.name = org_name
event_changed = 1
event.add_attribute("AS", str(ASN)) # each event has always one ASN
print("Creating new Event: " + event.uuid + " " + event.info)
# if event has attributes iterate them and see what changed
blocks = []
for att in event['Attribute']:
if att['type'] == 'AS':
print("Attribute ASN found: " + att['value'])
if att['type'] == 'ip-dst':
blocks.append(att['value'])
for field in fields: # Add CIDR blocks
if field not in blocks:
event.add_attribute("ip-dst", str(field))
"Requires at least two arguments: %s <JSON file> <create organisation> [<misp event title>]"
% sys.argv[0])
sys.exit()
if len(sys.argv) == 4:
event_import_info = sys.argv[3]
else:
event_import_info = False
json_import = sys.argv[1]
event_import_org = sys.argv[2]
event_import_uuid = str(uuid.uuid4()) # Unique ID
event_import_date = date.today() # Create event with current data
event_import_distribution = 2 # Connected
# Check if organisation already exist
org = MISPOrganisation()
try:
org.id = api.get_organisation(event_import_org, pythonify=True).id
except:
# We need to create a new one
org_new = MISPOrganisation()
org_new.name = event_import_org
org_new.uuid = str(uuid.uuid4())
org_new.type = "CSIRT"
org_new.sector = "Government"
org.id = api.add_organisation(org_new, pythonify=True).id
# Create the MISP event by loading the JSON file
# This will not add the attributes, but does add the event tags and galaxies
# We also add a random UUID for uniqueness
event = MISPEvent()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from pymisp import ExpandedPyMISP, MISPOrganisation
from keys import misp_url, misp_key, misp_verifycert
import argparse
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description=
'Edit the email of the organisation designed by the organisation_id.')
parser.add_argument(
"-i",
"--organisation_id",
required=True,
help=
"The name of the json file describing the organisation you want to modify."
)
parser.add_argument("-e",
"--email",
help="Email linked to the organisation.")
args = parser.parse_args()
misp = ExpandedPyMISP(misp_url, misp_key, misp_verifycert)
org = MISPOrganisation()
org.id = args.organisation_id
org.email = args.email
print(misp.update_organisation(org, pythonify=True))
