class TestBasic(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.misp = PyMISP(url, key, True, 'json', True)
def _clean_event(self, event):
event['Event'].pop('orgc_id', None)
event['Event'].pop('uuid', None)
event['Event'].pop('sharing_group_id', None)
event['Event'].pop('timestamp', None)
event['Event'].pop('org_id', None)
event['Event'].pop('date', None)
event['Event'].pop('RelatedEvent', None)
event['Event'].pop('publish_timestamp', None)
if event['Event'].get('Attribute'):
for a in event['Event'].get('Attribute'):
a.pop('uuid', None)
a.pop('event_id', None)
a.pop('id', None)
a.pop('timestamp', None)
if event['Event'].get('Orgc'):
event['Event']['Orgc'].pop('uuid', None)
event['Event']['Orgc'].pop('id', None)
if event['Event'].get('Org'):
event['Event']['Org'].pop('uuid', None)
event['Event']['Org'].pop('id', None)
return event['Event'].pop('id', None)
def new_event(self):
event = self.misp.new_event(0, 1, 0, "This is a test")
event_id = self._clean_event(event)
to_check = {
u'Event': {
u'info': u'This is a test',
u'locked': False,
u'attribute_count': u'0',
'disable_correlation': False,
u'analysis': u'0',
u'ShadowAttribute': [],
u'published': False,
u'distribution': u'0',
u'event_creator_email': u'*****@*****.**',
u'Attribute': [],
u'proposal_email_lock': False,
u'Org': {
u'name': u'ORGNAME'
},
u'Orgc': {
u'name': u'ORGNAME'
},
u'Galaxy': [],
u'threat_level_id': u'1'
}
}
self.assertEqual(event, to_check, 'Failed at creating a new Event')
return int(event_id)
def add_hashes(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.add_hashes(
event, 'Payload installation', 'dll_installer.dll',
'0a209ac0de4ac033f31d6ba9191a8f7a',
'1f0ae54ac3f10d533013f74f48849de4e65817a7',
'003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
'Fanny modules', False, 2)
self._clean_event(event)
to_check = {
u'Event': {
u'info':
u'This is a test',
u'locked':
False,
u'attribute_count':
u'3',
u'analysis':
u'0',
u'ShadowAttribute': [],
u'published':
False,
u'distribution':
u'0',
u'event_creator_email':
u'*****@*****.**',
u'Org': {
u'name': u'ORGNAME'
},
u'Orgc': {
u'name': u'ORGNAME'
},
u'Galaxy': [],
u'Attribute': [{
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|md5'
}, {
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|sha1'
}, {
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|sha256'
}],
u'proposal_email_lock':
False,
u'threat_level_id':
u'1'
}
}
self.assertEqual(event, to_check, 'Failed at adding hashes')
def publish(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.publish(event)
self._clean_event(event)
to_check = {
u'Event': {
u'info':
u'This is a test',
u'locked':
False,
u'attribute_count':
u'3',
u'analysis':
u'0',
u'ShadowAttribute': [],
u'published':
True,
u'distribution':
u'0',
u'event_creator_email':
u'*****@*****.**',
u'Org': {
u'name': u'ORGNAME'
},
u'Orgc': {
u'name': u'ORGNAME'
},
u'Galaxy': [],
u'Attribute': [{
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|md5'
}, {
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|sha1'
}, {
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|sha256'
}],
u'proposal_email_lock':
False,
u'threat_level_id':
u'1'
}
}
self.assertEqual(event, to_check, 'Failed at publishing event')
def delete(self, eventid):
event = self.misp.delete_event(eventid)
print(event)
def delete_attr(self, attrid):
event = self.misp.delete_attribute(attrid)
print(event)
def get(self, eventid):
event = self.misp.get_event(eventid)
print(event)
def get_stix(self, **kwargs):
event = self.misp.get_stix(kwargs)
print(event)
def add(self):
event = {
u'Event': {
u'info':
u'This is a test',
u'locked':
False,
u'attribute_count':
u'3',
u'analysis':
u'0',
u'ShadowAttribute': [],
u'published':
False,
u'distribution':
u'0',
u'event_creator_email':
u'*****@*****.**',
u'Attribute': [{
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|md5'
}, {
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|sha1'
}, {
u'category': u'Payload installation',
u'comment': u'Fanny modules',
u'to_ids': False,
u'value':
u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [],
u'distribution': u'2',
u'type': u'filename|sha256'
}],
u'proposal_email_lock':
False,
u'threat_level_id':
u'1'
}
}
event = self.misp.add_event(event)
print(event)
def add_user(self):
email = '*****@*****.**'
role_id = '5'
org_id = '1'
password = '******'
external_auth_required = False
external_auth_key = ''
enable_password = False
nids_sid = '1238717'
server_id = '1'
gpgkey = ''
certif_public = ''
autoalert = False
contactalert = False
disabled = False
change_pw = '0'
termsaccepted = False
newsread = '0'
authkey = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
to_check = {
'User': {
'email': email,
'org_id': org_id,
'role_id': role_id,
'password': password,
'external_auth_required': external_auth_required,
'external_auth_key': external_auth_key,
'enable_password': enable_password,
'nids_sid': nids_sid,
'server_id': server_id,
'gpgkey': gpgkey,
'certif_public': certif_public,
'autoalert': autoalert,
'contactalert': contactalert,
'disabled': disabled,
'change_pw': change_pw,
'termsaccepted': termsaccepted,
'newsread': newsread,
'authkey': authkey
}
}
user = self.misp.add_user(
email=email,
role_id=role_id,
org_id=org_id,
password=password,
external_auth_required=external_auth_required,
external_auth_key=external_auth_key,
enable_password=enable_password,
nids_sid=nids_sid,
server_id=server_id,
gpgkey=gpgkey,
certif_public=certif_public,
autoalert=autoalert,
contactalert=contactalert,
disabled=disabled,
change_pw=change_pw,
termsaccepted=termsaccepted,
newsread=newsread,
authkey=authkey)
# delete user to allow reuse of test
uid = user.get('User').get('id')
self.misp.delete_user(uid)
# ----------------------------------
# test interesting keys only (some keys are modified(password) and some keys are added (lastlogin)
tested_keys = [
'email', 'org_id', 'role_id', 'server_id', 'autoalert', 'authkey',
'gpgkey', 'certif_public', 'nids_sid', 'termsaccepted', 'newsread',
'contactalert', 'disabled'
]
for k in tested_keys:
self.assertEqual(
user.get('User').get(k),
to_check.get('User').get(k),
"Failed to match input with output on key: {}".format(k))
def add_organisation(self):
name = 'Organisation tests'
description = 'This is a test organisation'
orgtype = 'Type is a string'
nationality = 'French'
sector = 'Bank sector'
uuid = '16fd2706-8baf-433b-82eb-8c7fada847da'
contacts = 'Text field with no limitations'
local = False
to_check = {
'Organisation': {
'name': name,
'description': description,
'type': orgtype,
'nationality': nationality,
'sector': sector,
'uuid': uuid,
'contacts': contacts,
'local': local
}
}
org = self.misp.add_organisation(
name=name,
description=description,
type=orgtype,
nationality=nationality,
sector=sector,
uuid=uuid,
contacts=contacts,
local=local,
)
# delete organisation to allow reuse of test
oid = org.get('Organisation').get('id')
self.misp.delete_organisation(oid)
# ----------------------------------
tested_keys = [
'anonymise', 'contacts', 'description', 'local', 'name',
'nationality', 'sector', 'type', 'uuid'
]
for k in tested_keys:
self.assertEqual(
org.get('Organisation').get(k),
to_check.get('Organisation').get(k),
"Failed to match input with output on key: {}".format(k))
def test_create_event(self):
eventid = self.new_event()
time.sleep(1)
self.delete(eventid)
def test_get_event(self):
eventid = self.new_event()
time.sleep(1)
self.get(eventid)
time.sleep(1)
self.delete(eventid)
def test_add_event(self):
self.add()
time.sleep(1)
self.delete(1)
def test_del_attr(self):
eventid = self.new_event()
time.sleep(1)
self.delete_attr(1)
time.sleep(1)
self.delete(eventid)
def test_one_or_more(self):
self.assertEqual(self.misp._one_or_more(1), (1, ))
self.assertEqual(self.misp._one_or_more([1]), [1])
def test_create_user(self):
self.add_user()
def test_create_organisation(self):
self.add_organisation()
class MISPInstance():
owner_orgname: str
site_admin: PyMISP
_owner_site_admin: Optional[PyMISP] = None
_owner_orgadmin: Optional[PyMISP] = None
@property
def host_org(self) -> MISPOrganisation:
organisation = MISPOrganisation()
organisation.name = self.config['admin_orgname']
return self.create_or_update_organisation(organisation)
@property
def owner_site_admin(self) -> PyMISP:
if self._owner_site_admin:
return self._owner_site_admin
for user in self.site_admin.users():
if user.email == self.config['email_site_admin']:
break
else:
# The user doesn't exists
user = MISPUser()
user.email = self.config['email_site_admin']
user.org_id = self.host_org.id
user.role_id = 1 # Site admin
user = create_or_update_site_admin(self.site_admin, user)
user.authkey = self.config.get('site_admin_authkey')
dump_config = False
if not user.authkey: # type: ignore
dump_config = True
user.authkey = self.site_admin.get_new_authkey(user)
self.config['site_admin_authkey'] = user.authkey # type: ignore
user.password = self.config.get('site_admin_password')
if not user.password:
dump_config = True
if user.change_pw in ['1', True, 1]: # type: ignore
# Only change the password if the user never logged in.
user.password = ''.join(random.choices(string.ascii_uppercase + string.digits, k=16))
self.site_admin.update_user({'password': user.password, 'change_pw': 0}, user.id) # type: ignore
else:
user.password = '******'
self.config['site_admin_password'] = user.password
self._owner_site_admin = PyMISP(self.baseurl, user.authkey, # type: ignore
ssl=secure_connection, debug=False, timeout=300)
self._owner_site_admin.toggle_global_pythonify()
if dump_config:
with self.config_file.open('w') as f:
json.dump(self.config, f, indent=2)
return self._owner_site_admin
@property
def owner_orgadmin(self) -> PyMISP:
if self._owner_orgadmin:
return self._owner_orgadmin
for user in self.site_admin.users():
if user.email == self.config['email_orgadmin']:
break
else:
# The user doesn't exists
user = MISPUser()
user.email = self.config['email_orgadmin']
user.org_id = self.host_org.id
user.role_id = 2 # Site admin
user = self.create_or_update_user(user)
user.authkey = self.config.get('orgadmin_authkey')
dump_config = False
if not user.authkey: # type: ignore
dump_config = True
user.authkey = self.site_admin.get_new_authkey(user)
self.config['orgadmin_authkey'] = user.authkey # type: ignore
user.password = self.config.get('orgadmin_password')
if not user.password:
dump_config = True
if user.change_pw in ['1', True, 1]: # type: ignore
# Only change the password if the user never logged in.
user.password = ''.join(random.choices(string.ascii_uppercase + string.digits, k=16))
self.site_admin.update_user({'password': user.password, 'change_pw': 0}, user.id) # type: ignore
else:
user.password = '******'
self.config['orgadmin_password'] = user.password
# This user might have been disabled by the users
self._owner_orgadmin = PyMISP(self.baseurl, user.authkey, # type: ignore
ssl=secure_connection, debug=False, timeout=300)
self._owner_orgadmin.toggle_global_pythonify()
if dump_config:
with self.config_file.open('w') as f:
json.dump(self.config, f, indent=2)
return self._owner_orgadmin
def __init__(self, config_file: Path):
self.config_file = config_file
self.docker_compose_root = self.config_file.parent
with config_file.open() as f:
self.config = json.load(f)
self.owner_orgname = self.config['admin_orgname']
self.baseurl = self.config['baseurl']
self.hostname = self.config['hostname']
self.site_admin = PyMISP(self.baseurl, self.config['admin_key'],
ssl=secure_connection, debug=False, timeout=300)
self.site_admin.toggle_global_pythonify()
admin_user = self.site_admin.get_user()
self.site_admin.update_user({'change_pw': 0}, admin_user.id) # type: ignore
# Get container name
cur_dir = os.getcwd()
os.chdir(self.docker_compose_root)
command = shlex.split('sudo docker-compose ps -q misp')
p = Popen(command, stdout=PIPE, stderr=PIPE)
self.misp_container_name = p.communicate()[0].decode().strip()
# trash PyMISP so we can update
command = shlex.split('sudo docker-compose exec -T misp /bin/rm -rf /var/www/MISP/PyMISP')
Popen(command, stdout=PIPE, stderr=PIPE)
os.chdir(cur_dir)
# Make sure the external baseurl is set
self.update_external_baseurl(force=True)
# init the orgadmin (not site) user
self.owner_orgadmin
# Set the default role (id 3 is normal user)
self.owner_site_admin.set_default_role(3)
# Set the default sharing level to "All communities"
self.owner_site_admin.set_server_setting('MISP.default_event_distribution', 3, force=True)
# Enable taxonomies
self.enable_default_taxonomies()
# Set remaining config
self.owner_site_admin.set_server_setting('MISP.baseurl', self.baseurl, force=True)
self.owner_site_admin.set_server_setting('MISP.host_org_id', self.host_org.id)
self.owner_site_admin.set_server_setting('Security.rest_client_baseurl', 'http://127.0.0.1')
def pass_command_to_docker(self, command):
cur_dir = os.getcwd()
os.chdir(self.docker_compose_root)
c = shlex.split(command)
p = Popen(c, stdout=PIPE, stderr=PIPE)
to_return = p.communicate()
os.chdir(cur_dir)
return to_return
def copy_file(self, src, dst):
'''Copy/paste a file from HOST to the docker filesystem (MISP container)'''
return self.pass_command_to_docker(f'docker cp {src} {self.misp_container_name}:{dst}')
def update_external_baseurl(self, force: bool=False):
command = f'sudo docker inspect -f "{{{{.NetworkSettings.Networks.{internal_network_name}.IPAddress}}}}" {self.misp_container_name}'
outs, errs = self.pass_command_to_docker(command)
internal_ip = outs.strip().decode()
external_baseurl = f'http://{internal_ip}'
if force or external_baseurl != self.config['external_baseurl']:
self.config['external_baseurl'] = external_baseurl
self.update_misp_server_setting('MISP.external_baseurl', external_baseurl)
with self.config_file.open('w') as f:
json.dump(self.config, f, indent=2)
return external_baseurl
def enable_default_taxonomies(self):
for taxonomy in self.owner_site_admin.taxonomies():
if taxonomy.namespace in enabled_taxonomies:
self.owner_site_admin.enable_taxonomy(taxonomy)
def update_misp_server_setting(self, key, value):
return self.owner_site_admin.set_server_setting(key, value)
def change_session_timeout(self, timeout):
self.update_misp_server_setting('Session.timeout', timeout)
self.update_misp_server_setting('Session.cookieTimeout', timeout * 10)
def direct_call(self, url_path, payload=None):
return self.owner_site_admin.direct_call(url_path, payload)
def update_misp(self):
response = self.owner_site_admin.update_misp()
if response['results'][0]['status'] != 0:
print(json.dumps(response, indent=2))
def update_all_json(self):
self.owner_site_admin.update_object_templates()
self.owner_site_admin.update_galaxies()
self.owner_site_admin.update_taxonomies()
self.owner_site_admin.update_warninglists()
self.owner_site_admin.update_noticelists()
def sync_push_all(self):
for server in self.owner_site_admin.servers():
self.owner_site_admin.server_push(server)
def delete_events(self, events):
for e in events:
self.owner_site_admin.delete_event(e)
def create_or_update_user(self, user: MISPUser) -> MISPUser:
to_return_user = self.owner_site_admin.add_user(user)
if isinstance(to_return_user, MISPUser):
return to_return_user
# The user already exists
for u in self.owner_site_admin.users():
if u.email == user.email:
to_return_user = self.owner_site_admin.update_user(user, u.id) # type: ignore
if isinstance(to_return_user, MISPUser):
return to_return_user
raise Exception(f'Unable to update {user.email}: {to_return_user}')
else:
raise Exception(f'Unable to create {user.email}: {to_return_user}')
def create_or_update_tag(self, tag: MISPTag) -> MISPTag:
to_return_tag = self.owner_site_admin.add_tag(tag)
if isinstance(to_return_tag, MISPTag):
return to_return_tag
# The tag probably already exists
for t in self.owner_site_admin.tags():
if t.name == tag.name:
to_return_tag = self.owner_site_admin.update_tag(tag, t.id) # type: ignore
if isinstance(to_return_tag, MISPTag):
return to_return_tag
raise Exception(f'Unable to update {tag.name}: {to_return_tag}')
else:
raise Exception(f'Unable to create {tag.name}: {to_return_tag}')
def create_or_update_organisation(self, organisation: MISPOrganisation) -> MISPOrganisation:
to_return_org = self.site_admin.add_organisation(organisation)
if isinstance(to_return_org, MISPOrganisation):
return to_return_org
# The organisation is probably already there
for o in self.site_admin.organisations(scope='all'):
if o.name == organisation.name:
to_return_org = self.site_admin.update_organisation(organisation, o.id)
if isinstance(to_return_org, MISPOrganisation):
return self.site_admin.get_organisation(o.id) # type: ignore
raise Exception(f'Unable to update {organisation.name}: {to_return_org}')
else:
raise Exception(f'Unable to create {organisation.name}: {to_return_org}')
def init_default_user(self, email, password='******', role_id=1, org_id=None):
'''Default user is a local admin in the host org'''
user = MISPUser()
user.email = email
if org_id:
user.org_id = org_id
else:
for org in self.owner_site_admin.organisations():
if org.name == self.config['admin_orgname']:
user.org_id = org.id
break
else:
raise Exception('No default org found.')
user.role_id = role_id
user.password = password
self.create_or_update_user(user)
def user_statistics(self, context: str='data'):
return self.owner_site_admin.users_statistics(context)
def dump_all_events_as_feed(self, root_path: Path):
feed_dir = root_path / self.owner_orgname
feed_dir.mkdir(parents=True, exist_ok=True)
manifest = {}
hashes = []
for event in self.owner_site_admin.search(metadata=True):
e = self.owner_site_admin.get_event(event.uuid, deleted=True) # type: ignore
e_feed = e.to_feed(with_meta=True) # type: ignore
hashes += [[h, e.uuid] for h in e_feed['Event'].pop('_hashes')] # type: ignore
manifest.update(e_feed['Event'].pop('_manifest'))
with (feed_dir / f'{event.uuid}.json').open('w') as _fw: # type: ignore
json.dump(e_feed, _fw, indent=2)
with (feed_dir / 'hashes.csv').open('w') as hash_file:
for element in hashes:
hash_file.write('{},{}\n'.format(element[0], element[1]))
with (feed_dir / 'manifest.json').open('w') as manifest_file:
json.dump(manifest, manifest_file, indent=2)
def create_tag(self, name: str, exportable: bool, reserved: bool):
tag = MISPTag()
tag.name = name
tag.exportable = exportable
if reserved:
tag.org_id = self.host_org.id
self.create_or_update_tag(tag)
def __repr__(self):
return f'<{self.__class__.__name__}(external={self.baseurl})>'
# # Sync config
def create_sync_user(self, organisation, hostname):
self.sync_org = self.create_or_update_organisation(organisation)
email = f"sync_user@{hostname}"
user = MISPUser()
user.email = email
user.org_id = self.sync_org.id
user.role_id = 5 # Sync user
sync_user = self.create_or_update_user(user)
sync_user.authkey = self.owner_site_admin.get_new_authkey(sync_user)
sync_user_connector = PyMISP(self.owner_site_admin.root_url, sync_user.authkey, ssl=secure_connection, debug=False)
return sync_user_connector.get_sync_config(pythonify=True)
def configure_sync(self, server_sync_config, from_central_node=False):
# Add sharing server
for s in self.owner_site_admin.servers():
if s.name == server_sync_config.name:
server = s
break
else:
print(server_sync_config.to_json())
server = self.owner_site_admin.import_server(server_sync_config, pythonify=True)
server.pull = True
server.push = True # Not automatic, but allows to do a push
server.unpublish_event = unpublish_on_sync
server.url = server_sync_config.url # In case the internal IP changed, we want to update that.
server = self.owner_site_admin.update_server(server)
r = self.owner_site_admin.test_server(server)
if r['status'] != 1:
raise Exception(f'Sync test failed: {r}')
if from_central_node:
pull_to_create = tag_nodes_to_central
push_to_create = tag_central_to_nodes
else:
pull_to_create = tag_central_to_nodes
push_to_create = tag_nodes_to_central
pull_tags = []
push_tags = []
# The tags exist.
for tag in self.owner_site_admin.tags():
if tag.name in pull_to_create:
pull_tags.append(tag)
if tag.name in push_to_create:
push_tags.append(tag)
# Set limit on sync config
if push_tags:
# # Push
filter_tag_push = {"tags": {'OR': list(set([t.id for t in push_tags])), 'NOT': []}, 'orgs': {'OR': [], 'NOT': []}}
server.push_rules = json.dumps(filter_tag_push)
if pull_tags:
# # Pull
filter_tag_pull = {"tags": {'OR': list(set([t.name for t in pull_tags])), 'NOT': []}, 'orgs': {'OR': [], 'NOT': []}}
server.pull_rules = json.dumps(filter_tag_pull)
server = self.owner_site_admin.update_server(server)
# Add sharing group
for sg in self.owner_site_admin.sharing_groups():
if sg.name == f'Sharing group with {server_sync_config.Organisation["name"]}':
self.sharing_group = sg
break
else:
sharing_group = MISPSharingGroup()
sharing_group.name = f'Sharing group with {server_sync_config.Organisation["name"]}'
sharing_group.releasability = 'Training'
self.sharing_group = self.owner_site_admin.add_sharing_group(sharing_group)
self.owner_site_admin.add_server_to_sharing_group(self.sharing_group, server)
self.owner_site_admin.add_org_to_sharing_group(self.sharing_group, server_sync_config.Organisation)
self.owner_site_admin.add_org_to_sharing_group(self.sharing_group, self.host_org)
def __init__(self, misp_instance_dir: Path, secure_connection: bool):
with (misp_instance_dir / 'config.json').open() as f:
self.instance_config = json.load(f)
print('Initialize', self.instance_config['admin_orgname'])
self.secure_connection = secure_connection
self.synchronisations = {}
self.name = self.instance_config['admin_orgname']
# NOTE: never use that user again after initial config.
initial_user_connector = PyMISP(self.instance_config['baseurl'],
self.instance_config['admin_key'],
ssl=self.secure_connection,
debug=False)
# Set the default role (id 3 is normal user)
initial_user_connector.set_default_role(3)
initial_user_connector.toggle_global_pythonify()
self.baseurl = self.instance_config['baseurl']
self.external_baseurl = self.instance_config['external_baseurl']
# Create organisation
organisation = MISPOrganisation()
organisation.name = self.instance_config['admin_orgname']
self.host_org = initial_user_connector.add_organisation(organisation)
if not isinstance(self.host_org, MISPOrganisation):
# The organisation is probably already there
organisations = initial_user_connector.organisations()
for organisation in organisations:
if organisation.name == self.instance_config['admin_orgname']:
self.host_org = organisation
break
else:
raise Exception('Unable to find admin organisation')
# Create Site admin in new org
user = MISPUser()
user.email = self.instance_config['email_site_admin']
user.org_id = self.host_org.id
user.role_id = 1 # Site admin
self.host_site_admin = initial_user_connector.add_user(user)
if not isinstance(self.host_site_admin, MISPUser):
users = initial_user_connector.users()
for user in users:
if user.email == self.instance_config['email_site_admin']:
self.host_site_admin = user
break
else:
raise Exception('Unable to find admin user')
self.site_admin_connector = PyMISP(self.baseurl,
self.host_site_admin.authkey,
ssl=self.secure_connection,
debug=False)
self.site_admin_connector.toggle_global_pythonify()
# Setup external_baseurl
self.site_admin_connector.set_server_setting('MISP.external_baseurl',
self.external_baseurl,
force=True)
# Setup baseurl
self.site_admin_connector.set_server_setting('MISP.baseurl',
self.baseurl,
force=True)
# Setup host org
self.site_admin_connector.set_server_setting('MISP.host_org_id',
self.host_org.id)
# create other useful users
self.orgadmin = self.create_user(
self.instance_config['email_orgadmin'], 2)
self.user = self.create_user(self.instance_config['email_user'], 3)
# And connectors
self.org_admin_connector = PyMISP(self.baseurl,
self.orgadmin.authkey,
ssl=self.secure_connection,
debug=False)
self.org_admin_connector.toggle_global_pythonify()
self.user_connector = PyMISP(self.baseurl,
self.user.authkey,
ssl=self.secure_connection,
debug=False)
self.user_connector.toggle_global_pythonify()
class TestBasic(unittest.TestCase):
def setUp(self):
self.maxDiff = None
self.misp = PyMISP(url, key, True, 'json')
def _clean_event(self, event):
event['Event'].pop('orgc_id', None)
event['Event'].pop('uuid', None)
event['Event'].pop('sharing_group_id', None)
event['Event'].pop('timestamp', None)
event['Event'].pop('org_id', None)
event['Event'].pop('date', None)
event['Event'].pop('RelatedEvent', None)
event['Event'].pop('publish_timestamp', None)
if event['Event'].get('Attribute'):
for a in event['Event'].get('Attribute'):
a.pop('uuid', None)
a.pop('event_id', None)
a.pop('id', None)
a.pop('timestamp', None)
if event['Event'].get('Orgc'):
event['Event']['Orgc'].pop('uuid', None)
event['Event']['Orgc'].pop('id', None)
if event['Event'].get('Org'):
event['Event']['Org'].pop('uuid', None)
event['Event']['Org'].pop('id', None)
return event['Event'].pop('id', None)
def new_event(self):
event = self.misp.new_event(0, 1, 0, "This is a test")
event_id = self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'0', 'disable_correlation': False, u'analysis': u'0',
u'ShadowAttribute': [], u'published': False,
u'distribution': u'0', u'event_creator_email': u'*****@*****.**', u'Attribute': [], u'proposal_email_lock': False,
u'Object': [], u'Org': {u'name': u'ORGNAME'},
u'Orgc': {u'name': u'ORGNAME'},
u'Galaxy': [],
u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at creating a new Event')
return int(event_id)
def add_hashes(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.add_hashes(event, 'Payload installation', 'dll_installer.dll', '0a209ac0de4ac033f31d6ba9191a8f7a', '1f0ae54ac3f10d533013f74f48849de4e65817a7', '003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9', 'Fanny modules', False, 2)
self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'*****@*****.**',
u'Org': {u'name': u'ORGNAME'},
u'Orgc': {u'name': u'ORGNAME'},
u'Galaxy': [],
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at adding hashes')
def publish(self, eventid):
r = self.misp.get_event(eventid)
event = r.json()
event = self.misp.publish(event)
self._clean_event(event)
to_check = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': True, u'distribution': u'0', u'event_creator_email': u'*****@*****.**',
u'Org': {u'name': u'ORGNAME'},
u'Orgc': {u'name': u'ORGNAME'},
u'Galaxy': [],
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
self.assertEqual(event, to_check, 'Failed at publishing event')
def delete(self, eventid):
event = self.misp.delete_event(eventid)
print(event)
def delete_attr(self, attrid):
event = self.misp.delete_attribute(attrid)
print(event)
def get(self, eventid):
event = self.misp.get_event(eventid)
print(event)
def get_stix(self, **kwargs):
event = self.misp.get_stix(kwargs)
print(event)
def add(self):
event = {u'Event': {u'info': u'This is a test', u'locked': False,
u'attribute_count': u'3', u'analysis': u'0',
u'ShadowAttribute': [], u'published': False, u'distribution': u'0', u'event_creator_email': u'*****@*****.**',
u'Attribute': [
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|0a209ac0de4ac033f31d6ba9191a8f7a',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|md5'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|1f0ae54ac3f10d533013f74f48849de4e65817a7',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha1'},
{u'category': u'Payload installation', u'comment': u'Fanny modules',
u'to_ids': False, u'value': u'dll_installer.dll|003315b0aea2fcb9f77d29223dd8947d0e6792b3a0227e054be8eb2a11f443d9',
u'ShadowAttribute': [], u'distribution': u'2', u'type': u'filename|sha256'}],
u'proposal_email_lock': False, u'threat_level_id': u'1'}}
event = self.misp.add_event(event)
print(event)
def add_user(self):
email = '*****@*****.**'
role_id = '5'
org_id = '1'
password = '******'
external_auth_required = False
external_auth_key = ''
enable_password = False
nids_sid = '1238717'
server_id = '1'
gpgkey = ''
certif_public = ''
autoalert = False
contactalert = False
disabled = False
change_pw = '0'
termsaccepted = False
newsread = '0'
authkey = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
to_check = {'User': {'email': email, 'org_id': org_id, 'role_id': role_id,
'password': password, 'external_auth_required': external_auth_required,
'external_auth_key': external_auth_key, 'enable_password': enable_password,
'nids_sid': nids_sid, 'server_id': server_id, 'gpgkey': gpgkey,
'certif_public': certif_public, 'autoalert': autoalert,
'contactalert': contactalert, 'disabled': disabled,
'change_pw': change_pw, 'termsaccepted': termsaccepted,
'newsread': newsread, 'authkey': authkey}}
user = self.misp.add_user(email=email,
role_id=role_id,
org_id=org_id,
password=password,
external_auth_required=external_auth_required,
external_auth_key=external_auth_key,
enable_password=enable_password,
nids_sid=nids_sid,
server_id=server_id,
gpgkey=gpgkey,
certif_public=certif_public,
autoalert=autoalert,
contactalert=contactalert,
disabled=disabled,
change_pw=change_pw,
termsaccepted=termsaccepted,
newsread=newsread,
authkey=authkey)
# delete user to allow reuse of test
uid = user.get('User').get('id')
self.misp.delete_user(uid)
# ----------------------------------
# test interesting keys only (some keys are modified(password) and some keys are added (lastlogin)
tested_keys = ['email', 'org_id', 'role_id', 'server_id', 'autoalert',
'authkey', 'gpgkey', 'certif_public', 'nids_sid', 'termsaccepted',
'newsread', 'contactalert', 'disabled']
for k in tested_keys:
self.assertEqual(user.get('User').get(k), to_check.get('User').get(k), "Failed to match input with output on key: {}".format(k))
def add_organisation(self):
name = 'Organisation tests'
description = 'This is a test organisation'
orgtype = 'Type is a string'
nationality = 'French'
sector = 'Bank sector'
uuid = '16fd2706-8baf-433b-82eb-8c7fada847da'
contacts = 'Text field with no limitations'
local = False
to_check = {'Organisation': {'name': name, 'description': description,
'type': orgtype, 'nationality': nationality,
'sector': sector, 'uuid': uuid, 'contacts': contacts,
'local': local}}
org = self.misp.add_organisation(name=name,
description=description,
type=orgtype,
nationality=nationality,
sector=sector,
uuid=uuid,
contacts=contacts,
local=local,
)
# delete organisation to allow reuse of test
oid = org.get('Organisation').get('id')
self.misp.delete_organisation(oid)
# ----------------------------------
tested_keys = ['anonymise', 'contacts', 'description', 'local', 'name',
'nationality', 'sector', 'type', 'uuid']
for k in tested_keys:
self.assertEqual(org.get('Organisation').get(k), to_check.get('Organisation').get(k), "Failed to match input with output on key: {}".format(k))
def test_create_event(self):
eventid = self.new_event()
time.sleep(1)
self.delete(eventid)
def test_get_event(self):
eventid = self.new_event()
time.sleep(1)
self.get(eventid)
time.sleep(1)
self.delete(eventid)
def test_add_event(self):
self.add()
time.sleep(1)
self.delete(1)
def test_del_attr(self):
eventid = self.new_event()
time.sleep(1)
self.delete_attr(1)
time.sleep(1)
self.delete(eventid)
def test_one_or_more(self):
self.assertEqual(self.misp._one_or_more(1), (1,))
self.assertEqual(self.misp._one_or_more([1]), [1])
def test_create_user(self):
self.add_user()
def test_create_organisation(self):
self.add_organisation()
class MISPInstance():
def __init__(self, misp_instance_dir: Path, secure_connection: bool):
with (misp_instance_dir / 'config.json').open() as f:
self.instance_config = json.load(f)
print('Initialize', self.instance_config['admin_orgname'])
self.secure_connection = secure_connection
self.synchronisations = {}
self.name = self.instance_config['admin_orgname']
# NOTE: never use that user again after initial config.
initial_user_connector = PyMISP(self.instance_config['baseurl'],
self.instance_config['admin_key'],
ssl=self.secure_connection,
debug=False)
# Set the default role (id 3 is normal user)
initial_user_connector.set_default_role(3)
initial_user_connector.toggle_global_pythonify()
self.baseurl = self.instance_config['baseurl']
self.external_baseurl = self.instance_config['external_baseurl']
# Create organisation
organisation = MISPOrganisation()
organisation.name = self.instance_config['admin_orgname']
self.host_org = initial_user_connector.add_organisation(organisation)
if not isinstance(self.host_org, MISPOrganisation):
# The organisation is probably already there
organisations = initial_user_connector.organisations()
for organisation in organisations:
if organisation.name == self.instance_config['admin_orgname']:
self.host_org = organisation
break
else:
raise Exception('Unable to find admin organisation')
# Create Site admin in new org
user = MISPUser()
user.email = self.instance_config['email_site_admin']
user.org_id = self.host_org.id
user.role_id = 1 # Site admin
self.host_site_admin = initial_user_connector.add_user(user)
if not isinstance(self.host_site_admin, MISPUser):
users = initial_user_connector.users()
for user in users:
if user.email == self.instance_config['email_site_admin']:
self.host_site_admin = user
break
else:
raise Exception('Unable to find admin user')
self.site_admin_connector = PyMISP(self.baseurl,
self.host_site_admin.authkey,
ssl=self.secure_connection,
debug=False)
self.site_admin_connector.toggle_global_pythonify()
# Setup external_baseurl
self.site_admin_connector.set_server_setting('MISP.external_baseurl',
self.external_baseurl,
force=True)
# Setup baseurl
self.site_admin_connector.set_server_setting('MISP.baseurl',
self.baseurl,
force=True)
# Setup host org
self.site_admin_connector.set_server_setting('MISP.host_org_id',
self.host_org.id)
# create other useful users
self.orgadmin = self.create_user(
self.instance_config['email_orgadmin'], 2)
self.user = self.create_user(self.instance_config['email_user'], 3)
# And connectors
self.org_admin_connector = PyMISP(self.baseurl,
self.orgadmin.authkey,
ssl=self.secure_connection,
debug=False)
self.org_admin_connector.toggle_global_pythonify()
self.user_connector = PyMISP(self.baseurl,
self.user.authkey,
ssl=self.secure_connection,
debug=False)
self.user_connector.toggle_global_pythonify()
def __repr__(self):
return f'<{self.__class__.__name__}(external={self.baseurl})>'
def create_user(self, email, role_id):
user = MISPUser()
user.email = email
user.org_id = self.host_org.id
user.role_id = role_id
new_user = self.site_admin_connector.add_user(user)
if not isinstance(new_user, MISPUser):
users = self.site_admin_connector.users()
for user in users:
if user.email == email:
new_user = user
break
else:
raise Exception('Unable to find admin user')
return new_user
def create_sync_user(self, organisation: MISPOrganisation) -> MISPServer:
sync_org = self.site_admin_connector.add_organisation(organisation)
if not isinstance(sync_org, MISPOrganisation):
# The organisation is probably already there
organisations = self.site_admin_connector.organisations(
scope='all')
for org in organisations:
if org.name == organisation.name:
if not org.local:
org.local = True
org = self.site_admin_connector.update_organisation(
org)
sync_org = org
break
else:
raise Exception('Unable to find sync organisation')
short_org_name = sync_org.name.lower().replace(' ', '-')
email = f"sync_user@{short_org_name}.local"
user = MISPUser()
user.email = email
user.org_id = sync_org.id
user.role_id = 5 # Sync user
sync_user = self.site_admin_connector.add_user(user)
if not isinstance(sync_user, MISPUser):
users = self.site_admin_connector.users()
for user in users:
if user.email == email:
sync_user = user
break
else:
raise Exception('Unable to find sync user')
sync_user_connector = PyMISP(self.site_admin_connector.root_url,
sync_user.authkey,
ssl=self.secure_connection,
debug=False)
return sync_user_connector.get_sync_config(pythonify=True)
def configure_sync(self, server_sync_config: MISPServer):
# Add sharing server
for s in self.site_admin_connector.servers():
if s.name == server_sync_config.name:
server = s
break
else:
server = self.site_admin_connector.import_server(
server_sync_config)
server.pull = True
server.push = False
server = self.site_admin_connector.update_server(server)
r = self.site_admin_connector.test_server(server)
if r['status'] != 1:
raise Exception(f'Sync test failed: {r}')
print(server)
print(server.to_json(indent=2))
# NOTE: this is dirty.
self.synchronisations[server_sync_config.name.replace(
'Sync with ', '')] = server
def add_tag_filter_sync(self, server_sync: MISPServer, name: str):
# Add tag to limit push
tag = MISPTag()
tag.name = name
tag.exportable = False
tag.org_id = self.host_org.id
tag = self.site_admin_connector.add_tag(tag)
if not isinstance(tag, MISPTag):
for t in self.site_admin_connector.tags():
if t.name == name:
tag = t
break
else:
raise Exception('Unable to find tag')
# Set limit on sync config
filter_tag_push = {
"tags": {
'OR': [tag.id],
'NOT': []
},
'orgs': {
'OR': [],
'NOT': []
}
}
# filter_tag_pull = {"tags": {'OR': [], 'NOT': []}, 'orgs': {'OR': [], 'NOT': []}}
server_sync.push_rules = json.dumps(filter_tag_push)
# server.pull_rules = json.dumps(filter_tag_pull)
server_sync = self.site_admin_connector.update_server(server_sync)
def add_sharing_group(self,
name: str,
releasibility: str = 'Whatever it is a test',
servers: List[MISPServer] = [],
organisations: List[MISPOrganisation] = []):
# Add sharing group
for sg in self.site_admin_connector.sharing_groups():
if sg.name == name:
self.sharing_group = sg
break
else:
sharing_group = MISPSharingGroup()
sharing_group.name = name
sharing_group.releasability = releasibility
self.sharing_group = self.site_admin_connector.add_sharing_group(
sharing_group)
for server in servers:
self.site_admin_connector.add_server_to_sharing_group(
self.sharing_group, server)
for organisation in organisations:
self.site_admin_connector.add_org_to_sharing_group(
self.sharing_group, organisation)