class BaseView(object):
def __init__(self, config):
"""
Creator
:param config: The configuration for this module
:type config: Configuration
:returns: BaseView
"""
self.config = config
self.__logger = Log(config)
self.event_controller = EventController(config)
self.user_controller = UserController(config)
def get_json(self):
json = {}
cl = cherrypy.request.headers.get('Content-Length', None)
if cl:
try:
rawbody = cherrypy.request.body.read(int(cl))
json = loads(rawbody)
except TypeError:
# wonder what's wrong restangular?!?
counter = 0
rawbody = ''
while True:
rawbody += cherrypy.request.body.readline(counter)
try:
json = loads(rawbody)
break
except ValueError:
counter += 1
return json
@property
def logger(self):
return self.__logger.get_logger(self.__class__.__name__)
def _create_session(self):
"""
creates a session in cherrypy
"""
session = self._get_session()
session.regenerate()
self.logger.debug('Created a session')
def _put_to_session(self, key, value):
"""
puts/sets a key value pair to the session
:param key: The key for the value
:type key: object
:param value: The value for the key
:type value: object
"""
session = self._get_session()
session[key] = value
self.logger.debug('Set session value {0} for key {1}'.format(
value, key))
def __is_session_key(self, key):
"""
Checks if the key is existing the session, else raises a SessionNotFoundException
:param key: The key for the value
:type key: object
"""
session = self._get_session()
if key not in session.keys():
self.logger.debug('Key {0} is not defined in session'.format(key))
raise SessionNotFoundException(
'Key {0} was not defined in session'.format(key))
def _get_from_session(self, key, default_value=None):
"""
Get a variable by key from the session
Note: The variable stays in the session
:param key: The key for the value
:type key: object
:param value: The value for the key
:type value: object
:returns: object
"""
session = self._get_session()
value = session.get(key, default_value)
self.logger.debug('Returned session value "{0}" for key "{1}"'.format(
value, key))
return value
def _pull_from_session(self, key, default_value=None):
"""
Pulls a variable by key from the session
Note: The variable is removed from the session
:param key: The key for the value
:type key: object
:param value: The value for the key
:type value: object
:returns: object
"""
session = self._get_session()
value = session.pop(key, default_value)
self.logger.debug(
'Returned session value "{0}" for key "{1}" and removed it'.format(
value, key))
return value
def _destroy_session(self):
"""
Destroys a session
"""
try:
session = self._get_session()
session.clear()
session.delete()
# session.clean_up()
self.logger.debug('Session destroyed')
except CherryPyException as error:
self.logger.error(error)
def _get_session(self):
"""
Returns the session
"""
session = getattr(cherrypy, 'session')
self.logger.debug('Session returned')
return session
def user_authenticated(self):
username = self._get_from_session(SESSION_KEY, None)
return username
def get_user(self):
"""
Returns the user from the session
:returns: User
"""
user = self._get_from_session(SESSION_USER)
if user:
user = self.user_controller.get_user_by_username(user.username)
return user
def get_authorized_events_cache(self):
"""
Returns the authorized cached events
"""
try:
return self._get_from_session('_cp_events_cache', dict())
except AttributeError:
return dict()
def set_authorized_events_cache(self, cache):
try:
self._put_to_session('_cp_events_cache', cache)
except AttributeError:
pass
def check_if_admin(self):
user = self.get_user()
isadmin = is_user_priviledged(user)
if not isadmin:
raise cherrypy.HTTPError(
403, 'User {0} is not privileged'.format(user.username))
def check_if_admin_validate(self):
user = self.get_user()
if not user.permissions.validate:
raise cherrypy.HTTPError(
403, 'User {0} cannot validate events'.format(user.username))
def is_event_viewable(self, event, user=None):
# The same is in the mailer
if not user:
user = self.get_user()
if self.is_event_owner(event, user):
return True
else:
if user:
if user.group_id:
user_group = self.event_controller.group_broker.get_by_id(
user.group_id)
tlp_lvl = get_max_tlp(user_group)
if event.tlp_level_id >= tlp_lvl:
return True
else:
grp_ids = list()
for group in user_group.children:
grp_ids.append(group.identifier)
grp_ids.append(user.group_id)
for eventgroup in event.groups:
group = eventgroup.group
if group.identifier in grp_ids:
return True
return False
else:
return False
else:
return False
def check_if_event_is_viewable(self, event):
user = self.get_user()
if not self.is_event_viewable(event, user):
raise cherrypy.HTTPError(
403, 'User/group {0}/{1} cannot view this event'.format(
user.username, user.group.name))
def is_item_viewable(self, event, item, user=None):
if not user:
user = self.get_user()
if self.is_event_owner(event, user):
return True
else:
# check is the event is viewable then process to the iem
if self.is_event_viewable(event, user):
permissions = self.get_event_user_permissions(event, user)
if is_object_viewable(item, permissions, user):
return True
else:
return False
else:
return False
def check_item_is_viewable(self, event, item):
user = self.get_user()
result = self.is_item_viewable(event, item)
# if the result is still not set throw an error
log_msg = get_item_view_message(result, event.identifier,
item.identifier, user.username,
'can_view')
# update cache
self.logger.info(log_msg)
if result is None:
raise Exception(u'Unknown error occurred during event checks')
else:
return result
def check_if_event_is_modifiable(self, event):
self.check_permission(event, 'can_modify')
def check_if_event_is_deletable(self, event):
self.check_permission(event, 'can_delete')
def check_if_event_group_can_change(self, event):
self.check_permission(event, 'set_groups')
def check_permission(self, event, permission):
user = self.get_user()
result = self.is_user_allowed_to_perform(event, permission, user)
if not result:
raise cherrypy.HTTPError(
403,
'User/group {0}/{1} is not authorized perform action "{3}" on event {2}'
.format(user.username, user.group.name, event.identifier,
permission))
def check_if_owner(self, event):
user = self.get_user()
owner = is_event_owner(event, user)
if not owner:
raise cherrypy.HTTPError(
403, 'User/group {0}/{1} does not own event {2}'.format(
user.username, user.group.name, event.identifier))
def check_if_user_can_add(self, event):
user = self.get_user()
result = is_event_owner(event, user)
if not result:
result = self.is_user_allowed_to_perform(event, 'can_add', user)
if not result:
result = self.is_user_allowed_to_perform(
event, 'can_propose', user)
if not result:
raise cherrypy.HTTPError(
403, 'User {0}/{1} can not add contents to event {2}'.format(
user.username, user.group.name, event.identifier))
def is_user_allowed_to_perform(self, event, permission, user):
permissions = self.get_event_user_permissions(event, user)
if permissions:
result = getattr(permissions, permission)
# if the result is still not set throw an error
log_msg = get_view_message(result, event.identifier, user.username,
permission)
# update cache
self.logger.info(log_msg)
if result is None:
raise Exception(u'Unknown error occurred during event checks')
else:
return result
else:
return False
def is_user_priviledged(self, user):
self.logger.debug(u'Checking if user {0} is privileged'.format(
user.username))
result = is_user_priviledged(user)
if result:
self.logger.info(u'User {0} is privileged'.format(user.username))
else:
self.logger.info(u'User {0} is not privileged'.format(
user.username))
return result
def is_event_owner(self, event, user):
self.logger.debug(
u'Checking if user/group {0}/{1} owns event {2}'.format(
user.username, user.group.name, event.identifier))
result = is_event_owner(event, user)
if result:
self.logger.info(u'User/group {0}/{1} owns of event {2}'.format(
user.username, user.group.name, event.identifier))
else:
self.logger.info(
u'User/group {0}/{1} does not own of event {2}'.format(
user.username, user.group.name, event.identifier))
return result
def is_rest_insert(self, headers):
webinsert = headers.get('frontend', None)
if webinsert:
return False
else:
return True
def is_user_allowed_set_validate(self, event, old_instance, user, json):
properties = json.get('properties', None)
if properties:
permissions = self.get_event_user_permissions(event, user)
validated = properties.get('validated', None)
if validated is not None:
# validate is perhaps going to change
if permissions.can_validate:
return True
else:
# if there are changes deny them
if old_instance.properties.is_validated == validated:
# no change
return True
else:
self.logger.info(
u'User {0} has no right to validate elements of event {1}'
.format(user.username, event.identifier))
return False
return True
def is_user_allowed_set_share(self, event, old_instance, user, json):
properties = json.get('properties', None)
if properties:
shared = properties.get('shared', None)
if shared is not None:
if is_event_owner(event, user):
return True
else:
if old_instance.originating_group.identifier == user.group_id:
# User owns instance
return True
else:
if old_instance.properties.is_shareable == shared:
return True
else:
self.logger.info(
u'User/group {0}/{1} has no right to share elements of event {2}'
.format(user.username, user.group.name,
event.identifier))
return False
return True
def check_user_allowed_set_share(self, event, old_instance, user, json):
shared = self.is_user_allowed_set_share(event, old_instance, user,
json)
if not shared:
raise cherrypy.HTTPError(
403, 'User/group {0}/{1} cannot make this change'.format(
user.username, user.group.name))
def check_user_allowed_set_validate(self, event, old_instance, user, json):
validate = self.is_user_allowed_set_validate(event, old_instance, user,
json)
if not validate:
raise cherrypy.HTTPError(
403,
'User/group {0}/{1} can not validate elements of event {2}'.
format(user.username, user.group.name, event.identifier))
def is_if_user_can_set_validate_and_shared(self, event, old_instance, user,
json):
# TODO: Involve if user == partof the originator
validate = self.is_user_allowed_set_validate(event, old_instance, user,
json)
shared = self.is_user_allowed_set_validate(event, old_instance, user,
json)
return validate and shared
def check_if_user_can_set_validate_or_shared(self, event, old_instance,
user, json):
self.check_user_allowed_set_share(event, old_instance, user, json)
self.check_user_allowed_set_validate(event, old_instance, user, json)
def get_event_user_permissions(self, event, user):
cache = self.get_authorized_events_cache()
permissions = None
if event:
# check if cache
if cache:
permissions = cache.get(event.identifier, None)
# if None the event is not available inside the cache
if permissions:
return permissions
if permissions is None:
if self.is_event_owner(event, user):
permissions = EventPermissions('0')
permissions.set_all()
else:
# either there was no cache or the event was not inside the cache perform checks
permissions = self.event_controller.get_event_user_permissions(
event, user)
# put result in the cache if there is one
cache[event.identifier] = permissions
# perform checks
else:
permissions = EventPermissions('0')
self.set_authorized_events_cache(cache)
return permissions
def put_user_to_session(self, user):
cherrypy.request.login = user.username
self._put_to_session('_cp_username', user.username)
offline_user = self.__make_user_object(user)
self._put_to_session(SESSION_USER, offline_user)
def __make_user_object(self, user):
# TODO: make user offline
obj = GenObject()
obj.name = user.name
obj.username = user.username
obj.identifier = user.identifier
obj.email = user.email
obj.group_id = user.group_id
obj.activated = user.activated
obj.sirname = user.sirname
obj.permissions = UserRights(user.dbcode)
return obj
class Ce1susAdapter(BaseController):
def __init__(self, config, session=None):
self.server_details = None
self.proxies = {}
self.verify_ssl = False
self.ssl_cert = None
self.session = requests.session()
self.assembler = Assembler(config, session)
self.event_controller = EventController(config, session)
self.process_controller = ProcessController(config, session)
@property
def apiUrl(self):
return '{0}/REST/0.3.0'.format(self.server_details.baseurl)
@property
def apiKey(self):
return self.server_details.user.api_key
def __set_complete_inflated(self, url, complete=False, inflated=False):
if complete and not inflated:
url = '{0}?complete=true'.format(url)
if not complete and inflated:
url = '{0}?inflated=true'.format(url)
if complete and inflated:
url = '{0}?complete=true&inflated=true'.format(url)
return url
def __extract_message(self, error):
reason = error.message
message = error.response.text
code = error.response.status_code
# "<p>An event with uuid "54f63b0f-0c98-4e74-ab95-60c718689696" already exists</p>
try:
pos = message.index('<p>') + 3
message = message[pos:]
pos = message.index('</p>')
message = message[:pos]
except ValueError:
# In case the message is not parsable
pass
return code, reason, message
def __handle_exception(self, request):
try:
request.raise_for_status()
except requests.exceptions.HTTPError as error:
code, reason, message = self.__extract_message(error)
message = u'{0} ({1})'.format(reason, message)
if code == 403:
raise Ce1susAdapterForbiddenException(message)
elif code == 404:
raise Ce1susAdapterNothingFoundException(message)
else:
raise Ce1susAdapterException(message)
def __request(self, path, method, data=None, extra_headers=None):
try:
url = '{0}/{1}'.format(self.apiUrl, path)
headers = {
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'Ce1sus API server client {0}'.format(APP_REL),
'key': self.apiKey
}
if extra_headers:
for key, value in extra_headers.items():
headers[key] = value
if method == 'GET':
request = self.session.get(url,
headers=headers,
proxies=self.proxies,
verify=self.verify_ssl,
cert=self.ssl_cert,
cookies=self.session.cookies)
elif method == 'PUT':
request = self.session.put(url,
json.dumps(data),
headers=headers,
proxies=self.proxies,
verify=self.verify_ssl,
cert=self.ssl_cert,
cookies=self.session.cookies)
elif method == 'DELETE':
request = self.session.delete(url,
headers=headers,
proxies=self.proxies,
verify=self.verify_ssl,
cert=self.ssl_cert,
cookies=self.session.cookies)
elif method == 'POST':
request = self.session.post(url,
json.dumps(data),
headers=headers,
proxies=self.proxies,
verify=self.verify_ssl,
cert=self.ssl_cert,
cookies=self.session.cookies)
else:
raise UnkownMethodException(
u'Mehtod {0} is not specified can only be GET,POST,PUT or DELETE'
)
if request.status_code == requests.codes.ok:
return json.loads(request.text)
else:
self.__handle_exception(request)
except requests.exceptions.RequestException as error:
raise Ce1susAdapterException(error)
except requests.ConnectionError as error:
raise Ce1susAdapterConnectionException('{0}'.format(error.message))
def login(self):
text = self.__request('/login', 'POST', None)
return text
def logout(self):
text = self.__request(
'/logout',
'GET',
)
if text == 'User logged out':
return True
else:
return False
def get_event_by_uuid(self,
uuid,
complete=False,
inflated=False,
poponly=True,
json=False):
url = '/event/{0}'.format(uuid)
url = self.__set_complete_inflated(url, complete, inflated)
json = self.__request(url, 'GET', None)
if json:
return json
else:
event = self.assembler.assemble_event(json,
self.server_details.user,
False, True, poponly)
return event
def insert_event(self, event, complete=False, inflated=False):
url = '/event'
url = self.__set_complete_inflated(url, complete, inflated)
event_permissions = self.event_controller.get_event_user_permissions(
event, self.server_details.user)
json = self.__request(url,
'POST',
data=event.to_dict(True, True, event_permissions,
self.server_details.user))
owner = is_event_owner(event, self.server_details.user)
event = self.assembler.update_event(event, json,
self.server_details.user, owner,
True)
return event
def update_event(self, event, complete=False, inflated=False):
url = '/event/{0}'.format(event.uuid)
url = self.__set_complete_inflated(url, complete, inflated)
event_permissions = self.event_controller.get_event_user_permissions(
event, self.server_details.user)
json = self.__request(url,
'PUT',
data=event.to_dict(True, True, event_permissions,
self.server_details.user))
owner = is_event_owner(event, self.server_details.user)
event = self.assembler.update_event(event, json,
self.server_details.user, owner,
True)
return event
def get_index(self, server_details=None):
url = '/events'
if server_details:
self.server_details = server_details
url = self.__set_complete_inflated(url, False, False)
json = self.__request(url, 'GET', None)
events_json = json.get('data', list())
result = list()
for event_json in events_json:
event = self.assembler.assemble_event(event_json,
self.server_details.user,
False, True, True)
result.append(event)
return result
def push(self, server_details):
try:
self.server_details = server_details
self.login()
user_events = self.event_controller.get_all_for_user(
server_details.user)
# get the remote ones
rem_events = self.get_index(server_details)
rem_events_dict = dict()
for rem_event in rem_events:
rem_events_dict[rem_event.uuid] = rem_event
uuids_to_push = list()
for event in user_events:
rem_event = rem_events_dict.get(event.uuid, None)
if rem_event:
if event.last_publish_date and rem_event.last_publish_date:
if event.last_publish_date > rem_event.last_publish_date:
uuids_to_push.append(event.uuid)
else:
uuids_to_push.append(event.uuid)
# pass
for uuid_to_push in uuids_to_push:
self.process_controller.create_new_process(
ProcessType.PUSH, uuid_to_push, self.server_details.user,
server_details, True)
except Ce1susAdapterException as error:
self.logout()
raise Ce1susAdapterException(error)
self.logout()
def pull(self, server_details):
try:
self.server_details = server_details
self.login()
events = self.get_index(server_details)
event_uuids = dict()
for event in events:
event_uuids[event.uuid] = event
local_events = self.event_controller.get_event_by_uuids(
event_uuids.keys())
items_to_remove = list()
for local_event in local_events:
rem_event = event_uuids[local_event.uuid]
if local_event.last_publish_date and rem_event.last_publish_date:
if rem_event.last_publish_date <= local_event.last_publish_date:
items_to_remove.append(local_event.uuid)
else:
items_to_remove.append(local_event.uuid)
for item_to_remove in items_to_remove:
del event_uuids[item_to_remove]
for rem_event in event_uuids.itervalues():
self.process_controller.create_new_process(
ProcessType.PULL, rem_event.uuid, server_details.user,
server_details, True)
self.logout()
return 'OK'
except Ce1susAdapterException as error:
self.logout()
raise Ce1susAdapterException(error)
class Ce1susStixMapper(BaseController):
def __init__(self, config, session=None):
BaseController.__init__(self, config, session)
self.cybox_mapper = CyboxMapper(config, session)
self.indicator_controller = IndicatorController(config, session)
self.event_controller = EventController(config, session)
self.fh = FileHandler()
def init(self):
self.seen_groups = list()
self.seen_observables = list()
def __map_stix_header(self, event):
self.init()
stix_header = STIXHeader(title=event.title)
stix_header.description = event.description
stix_header.short_description = event.title
identifiy = self.create_stix_identity(event)
time = self.cybox_mapper.get_time(produced_time=event.created_at,
received_time=event.modified_on)
info_source = InformationSource(identity=identifiy, time=time)
stix_header.information_source = info_source
# Add TLP
marking = Marking()
marking_spec = MarkingSpecification()
marking.add_marking(marking_spec)
tlp_marking_struct = TLPMarkingStructure()
tlp_marking_struct.color = event.tlp.upper()
tlp_marking_struct.marking_model_ref = 'http://govcert.lu/en/docs/POL_202_V2.2_rfc2350.pdf'
marking_spec.marking_structures = list()
marking_spec.marking_structures.append(tlp_marking_struct)
stix_header.handling = marking
return stix_header
def create_stix_identity(self, obj):
idenfitier = 'ce1sus:Group-{0}'.format(obj.creator_group.uuid)
if idenfitier in self.seen_groups:
identity = Identity()
identity.idref = idenfitier
else:
identity = Identity()
identity.id_ = idenfitier
identity.name = obj.creator_group.name
self.seen_groups.append(idenfitier)
return identity
def add_custom_property(self, instance, attribute):
if not instance.custom_properties:
cust_props = CustomProperties()
instance.custom_properties = cust_props
custom_prop = Property()
custom_prop.name = attribute.definition.name
custom_prop.description = attribute.definition.description
custom_prop._value = attribute.value
if hasattr(custom_prop._value, 'condition'):
custom_prop._value.condition = self.get_condition(attribute)
if hasattr(custom_prop, 'condition'):
custom_prop.condition = self.get_condition(attribute)
instance.custom_properties.append(custom_prop)
def create_EmailAddress(self, attribute):
email = EmailAddress(attribute.value)
email.condition = self.get_condition(attribute)
return email
def __check_set_email_header(self, cybox_email):
if not cybox_email.header:
cybox_email.header = EmailHeader()
def populate_email(self, cybox_email, attribute):
# returns a cybox email object out of a ce1sus object
def_name = attribute.definition.name
if def_name == 'email_attachment_file_name':
attachment = File()
attachment.file_name = attribute.value
attachment.file_name.condition = self.get_condition(attribute)
# cybox_email.attachments = Attachments()
# cybox_email.attachments.append(File)
elif def_name == 'email_bcc':
self.__check_set_email_header(cybox_email)
if not cybox_email.header.bcc:
cybox_email.header.bcc = EmailRecipients()
cybox_email.header.bcc.append(self.create_EmailAddress(attribute))
elif def_name == 'email_cc':
self.__check_set_email_header(cybox_email)
if not cybox_email.header.cc:
cybox_email.header.cc = EmailRecipients()
cybox_email.header.bcc.append(self.create_EmailAddress(attribute))
elif def_name == 'email_errors_to':
self.__check_set_email_header(cybox_email)
self.set_check_attr(cybox_email, 'header.errors_to', attribute)
elif def_name == 'email_message_id':
self.__check_set_email_header(cybox_email)
self.set_check_attr(cybox_email, 'header.message_id', attribute)
elif def_name == 'email_mime_version':
self.__check_set_email_header(cybox_email)
self.set_check_attr(cybox_email, 'header.mime_version', attribute)
elif def_name == 'email_raw_body':
self.set_check_attr(cybox_email, 'raw_body', attribute)
elif def_name == 'email_raw_header':
self.set_check_attr(cybox_email, 'raw_header', attribute)
elif def_name == 'email_reply_to':
if not cybox_email.header.in_reply_to:
self.__check_set_email_header(cybox_email)
cybox_email.header.in_reply_to = EmailRecipients()
cybox_email.header.in_reply_to.append(
self.create_EmailAddress(attribute))
elif def_name == 'email_server':
self.set_check_attr(cybox_email, 'email_server', attribute)
elif def_name == 'email_subject':
self.set_check_attr(cybox_email, 'subject', attribute)
elif def_name == 'email_from':
self.__check_set_email_header(cybox_email)
if not cybox_email.header.from_:
cybox_email.header.from_ = self.create_EmailAddress(attribute)
elif def_name == 'email_to':
self.__check_set_email_header(cybox_email)
if not cybox_email.header.to:
cybox_email.header.to = EmailRecipients()
cybox_email.header.to.append(self.create_EmailAddress(attribute))
elif def_name == 'email_x_mailer':
self.set_check_attr(cybox_email, 'header.x_mailer', attribute)
elif def_name == 'email_x_originating_ip':
self.set_check_attr(cybox_email, 'header.x_originating_ip',
attribute)
elif 'hash' in def_name:
raise CyboxMapperException('Not defined')
elif def_name == 'email_link':
if not cybox_email.links:
cybox_email.links = Links()
cybox_email.links.append(Link(attribute.value))
elif def_name == 'email_send_date':
cybox_email.date = attribute.value
elif def_name == 'email_in_reply_to':
self.__check_set_email_header(cybox_email)
cybox_email.header.in_reply_to = attribute.value
else:
raise CyboxMapperException('Not defined for {0}'.format(def_name))
def set_check_attr(self, cybox_obj, proerty_name, attribute):
# TODO make this correct
cybox_value = None
if '.' in proerty_name:
properies = proerty_name.split('.')
prop1 = getattr(cybox_obj, properies[0])
if prop1:
cybox_value = getattr(prop1, properies[1])
else:
pass
else:
cybox_value = getattr(cybox_obj, proerty_name, None)
if cybox_value:
cybox_value.condition = self.get_condition(attribute)
else:
setattr(cybox_obj, proerty_name, attribute.value)
cybox_value = getattr(cybox_obj, proerty_name, None)
if hasattr(cybox_value, 'condition'):
cybox_value.condition = self.get_condition(attribute)
def map_attribtue(self, instance, attribute):
definition_name = attribute.definition.name
if hasattr(instance, definition_name.lower()):
setattr(instance, definition_name.lower(), attribute.value)
value = getattr(instance, definition_name.lower())
value.condition = self.get_condition(attribute)
else:
if 'hash' in definition_name:
if attribute.condition.value:
if attribute.condition.value:
if attribute.condition.value == "Equals":
exact = True
else:
exact = False
else:
exact = False
else:
exact = True
h = Hash(attribute.value, exact=exact)
h.condition = attribute.condition
if instance.hashes is None:
instance.hashes = HashList()
instance.hashes.append(h)
pass
elif 'email' in definition_name:
self.populate_email(instance, attribute)
elif definition_name == 'url':
instance.type_ = URI.TYPE_URL
instance.value = attribute.value
instance.value.condition = self.get_condition(attribute)
elif 'Full_Path' and isinstance(instance, Process):
# TODO: check why this is set?!?
pass
elif definition_name == 'WindowsRegistryKey_Key':
instance.key = attribute.value
instance.key.condition = self.get_condition(attribute)
elif definition_name == 'WindowsRegistryKey_Hive':
instance.hive = attribute.value
instance.hive.condition = self.get_condition(attribute)
elif 'WindowsRegistryKey_RegistryValue' in definition_name:
value = RegistryValue()
if definition_name == 'WindowsRegistryKey_RegistryValue_Data':
value.data = attribute.value
value.data.condition = self.get_condition(attribute)
elif definition_name == 'WindowsRegistryKey_RegistryValue_Data':
value.name = attribute.value
value.name.condition = self.get_condition(attribute)
if not instance.values:
instance.values = RegistryValues()
instance.values.append(value)
instance.data = attribute.value
elif definition_name == 'ipv4_addr':
instance.category = definition_name.replace('_', '-')
instance.address_value = attribute.value
instance.address_value.condition = self.get_condition(
attribute)
elif definition_name == 'DomainName_Value':
instance.value = attribute.value
elif definition_name == 'Raw_Artifact':
path = '{0}/{1}'.format(self.fh.get_base_path(),
attribute.value)
if isfile(path):
with open(path, 'r') as f:
bin_string = f.read()
instance.data = bin_string
# TODO find the corect type
instance.type_ = Artifact.TYPE_GENERIC
instance.packaging.append(Base64Encoding())
else:
instance.data = 'MIA'
instance.type_ = Artifact.TYPE_GENERIC
instance.packaging.append(Base64Encoding())
elif definition_name == 'content_type':
instance.type_ = attribute.value
elif definition_name == 'URIType':
instance.type_ = attribute.value
elif not attribute.definition.cybox_std:
self.add_custom_property(instance, attribute)
elif isinstance(instance,
NetworkConnection) and definition_name in [
'is_type', 'Port'
]:
# TODO: check why this is set?!?
pass
else:
raise Ce1susStixMapperException(
'Cannot map {1} on object type {0}'.format(
instance.__class__.__name__, definition_name))
def get_condition(self, attribute):
if attribute.condition:
return attribute.condition.value
else:
# default condition is equals
return 'Equals'
def create_object(self, ce1sus_object, event_permissions, user):
definition_name = ce1sus_object.definition.name
obj = Object()
identifier = 'ce1sus:Object-{0}'.format(ce1sus_object.uuid)
obj.id_ = identifier
# try to find automatic the object container
try:
clazz = get_class(
'cybox.objects.{0}_object'.format(definition_name.lower()),
definition_name)
instance = clazz()
if definition_name == 'Disk':
# TODO: check why this must be set stix bug?
setattr(instance, 'type_', None)
except ImportError:
if definition_name == 'WinEventLog':
instance = WinEventLog()
# TODO: check why this must be set stix bug?
setattr(instance, 'type_', None)
elif definition_name == 'UserAccount':
instance = UserAccount()
elif definition_name == 'WinService':
instance = WinService()
elif definition_name == 'WindowsRegistryKey':
instance = WinRegistryKey()
elif definition_name == 'NetworkConnection':
instance = NetworkConnection()
elif definition_name == 'WinVolume':
instance = WinVolume()
# TODO: check why this must be set stix bug?
setattr(instance, 'drive_type', None)
elif definition_name == 'WinKernelHook':
instance = WinKernelHook()
elif definition_name == 'WinDriver':
instance = WinDriver()
elif definition_name == 'DomainName':
instance = DomainName()
# TODO: try to map it manually
elif definition_name == 'email':
instance = EmailMessage()
else:
raise Ce1susStixMapperException(
'Required to map manually {0}'.format(definition_name))
obj.properties = instance
attributes = ce1sus_object.get_attributes_for_permissions(
event_permissions, user)
for attribute in attributes:
self.map_attribtue(instance, attribute)
rel_objects = ce1sus_object.get_related_objects_for_permissions(
event_permissions, user)
for rel_object in rel_objects:
ob = self.create_object(rel_object.object, event_permissions, user)
if ob:
rel_obj = self.create_related_object(rel_object,
event_permissions, user)
# cybox_rel_object = RelatedObject(properties=ob.properties, relationship=rel_object.relation)
obj.related_objects.append(rel_obj)
return obj
def create_related_object(self, rel_object, event_permissions, user):
ob = self.create_object(rel_object.object, event_permissions, user)
rel_obj = RelatedObject(properties=ob.properties,
relationship=rel_object.relation)
rel_obj.id_ = ob.id_
for relobj in rel_object.object.related_objects:
rel_child_obj = self.create_related_object(relobj,
event_permissions, user)
rel_obj.related_objects.append(rel_child_obj)
return rel_obj
def create_composed_observable(self, ce1sus_composition, event_permissions,
user):
composition = ObservableComposition()
composition.operator = ce1sus_composition.operator
ce1sus_obs = ce1sus_composition.get_observables_for_permissions(
event_permissions, user)
for ce1sus_ob in ce1sus_obs:
obs = self.create_observable(ce1sus_ob, event_permissions, user)
composition.observables.append(obs)
return composition
def create_observable(self, ce1sus_obs, event_permissions, user):
identifier = 'ce1sus:Observable-{0}'.format(ce1sus_obs.uuid)
cybox_observable = Observable()
if identifier in self.seen_observables:
# if I've seen the uuid then make a reference insead
cybox_observable.idref = identifier
else:
self.seen_observables.append(identifier)
cybox_observable.id_ = identifier
cybox_observable.title = ce1sus_obs.title
cybox_observable.description = ce1sus_obs.description
if ce1sus_obs.object:
cybox_obj = self.create_object(ce1sus_obs.object,
event_permissions, user)
cybox_observable.object_ = cybox_obj
elif ce1sus_obs.observable_composition:
cybox_obj_composition = self.create_composed_observable(
ce1sus_obs.observable_composition, event_permissions, user)
cybox_observable.observable_composition = cybox_obj_composition
return cybox_observable
def create_indicator(self, ce1sus_indicator, event_permissions, user):
indicator = Indicator()
indicator.id_ = 'ce1sus:Indicator-{0}'.format(ce1sus_indicator.uuid)
indicator.title = ce1sus_indicator.title
indicator.description = ce1sus_indicator.description
indicator.short_description = ce1sus_indicator.short_description
if ce1sus_indicator.confidence:
indicator.confidence = ce1sus_indicator.confidence.title()
else:
indicator.confidence = 'Low'
# TODO: handling
# TODO: markings
for type_ in ce1sus_indicator.types:
indicator.add_indicator_type(type_.name)
if ce1sus_indicator.operator:
indicator.observable_composition_operator = ce1sus_indicator.operator
# Todo Add confidence
# indicator_attachment.confidence = "Low"
creator = self.create_stix_identity(ce1sus_indicator)
time = self.cybox_mapper.get_time(
produced_time=ce1sus_indicator.created_at)
info_source = InformationSource(identity=creator, time=time)
indicator.producer = info_source
observables = ce1sus_indicator.get_observables_for_permissions(
event_permissions, user)
for obs in observables:
cybox_obs = self.create_observable(obs, event_permissions, user)
indicator.add_observable(cybox_obs)
valid_time = ValidTime(start_time=ce1sus_indicator.created_at,
end_time=ce1sus_indicator.created_at)
indicator.add_valid_time_position(valid_time)
return indicator
def create_stix(self, event, user):
stix_package = STIXPackage()
stix_package.id_ = 'ce1sus:Event-{0}'.format(event.uuid)
stix_header = self.__map_stix_header(event)
stix_package.stix_header = stix_header
event_permissions = self.event_controller.get_event_user_permissions(
event, user)
# observables
if event.observables:
for observable in event.get_observables_for_permissions(
event_permissions, user):
cybox_obs = self.create_observable(observable,
event_permissions, user)
stix_package.add_observable(cybox_obs)
# indicators
if event.indicators:
indicators = event.get_indicators_for_permissions(
event_permissions, user)
else:
# generate indicators
indicators = self.indicator_controller.get_generic_indicators(
event, user)
for indicator in indicators:
stix_indicator = self.create_indicator(indicator,
event_permissions, user)
stix_package.add_indicator(stix_indicator)
return stix_package
class MISPAdapter(BaseView):
def __init__(self, config, session=None):
BaseView.__init__(self, config)
self.event_controller = EventController(config, session)
self.server_broker = self.event_controller.broker_factory(
SyncServerBroker)
self.ce1sus_to_misp = Ce1susMISP(config, session)
self.login_handler = LoginHandler(config)
self.logout_handler = LogoutHandler(config)
self.misp_converter = MispConverter(config, None, None, None, session)
dump = config.get('MISPAdapter', 'dump', False)
file_loc = config.get('MISPAdapter', 'file', None)
self.misp_converter.dump = dump
self.misp_converter.file_location = file_loc
self.merger = Merger(config, session)
self.process_controller = ProcessController(config, session)
@cherrypy.expose
@cherrypy.tools.allow(methods=['POST'])
@require()
def upload_xml(self, *vpath, **params):
try:
input_json = self.get_json()
filename = input_json['name']
data = input_json['data']['data']
xml_string = base64.b64decode(data)
complete = params.get('complete', False)
inflated = params.get('inflated', False)
user = self.get_user()
user = self.user_controller.get_user_by_username(user.username)
if 'XML' in filename or 'xml' in filename:
self.logger.info(
'Starting to import xml form file {0}'.format(filename))
self.misp_converter.user = user
event_uuid = self.misp_converter.get_uuid_from_event_xml(
xml_string)
try:
event = self.misp_converter.get_event_from_xml(
xml_string, None)
self.logger.info('Received Event {0}'.format(event.uuid))
self.event_controller.insert_event(user, event, True, True)
except ControllerIntegrityException as error:
local_event = self.event_controller.get_event_by_uuid(
event_uuid)
event = self.misp_converter.get_event_from_xml(
xml_string, local_event)
# merge event with existing event
if self.is_event_viewable(local_event, user):
event_permissions = self.get_event_user_permissions(
event, user)
try:
merged_event = self.merger.merge_event(
local_event, event, user, event_permissions)
except MergingException:
raise MISPAdapterException405()
# raise cherrypy.HTTPError(405)
else:
# TODO log the changes
self.logger.warning(
'user {0} tried to change event {1} but does not have the right to see it'
.format(user.username, event.identifer))
raise HTTPError(403, 'Not authorized')
if merged_event:
self.logger.info('Received Event {0} updates'.format(
merged_event.uuid))
self.event_controller.update_event(
user, merged_event, True, True)
event = merged_event
else:
self.logger.info(
'Received Event {0} did not need to update as it is up to date'
.format(event.uuid))
# Log errors however
self.event_controller.event_broker.do_commit()
event_permissions = self.event_controller.get_event_user_permissions(
event, user)
cherrypy.response.headers[
'Content-Type'] = 'application/json; charset=UTF-8'
return event.to_dict(complete, inflated, event_permissions,
user)
else:
raise HTTPError(409, 'File does not end in xml or XML')
except MispConverterException as error:
self.logger.error(error)
raise HTTPError(409, 'File is not a MISP XML')
except ControllerException as error:
self.logger.error(error)
raise HTTPError(400, '{0}'.format(error.message))
except Exception as error:
self.logger.critical(error)
raise HTTPError(500, '{0}'.format(error.message))
@cherrypy.expose
@cherrypy.tools.allow(methods=['GET'])
def shadow_attributes(self, *vpath, **params):
# this is called from the misp server to see it his know events have new proposals
# TODO: Proposal for misp
raise cherrypy.HTTPError(404)
def get_all_events(self, server_user):
events = self.event_controller.get_all()
result = list()
for event in events:
if event.properties.is_validated:
if self.is_event_viewable(event, server_user):
result.append(event)
else:
if event.originating_group.identifier == server_user.group.identifier:
result.append(event)
return result
def make_index(self, server_user):
result = self.get_all_events(server_user)
return self.ce1sus_to_misp.make_index(result)
def pull(self, server_details):
# use the logged in user for these request
if server_details.type != 'MISP':
raise MISPAdapterException(
'Server {0} is not a MISP server'.format(
server_details.identifier))
self.misp_converter.api_key = server_details.user.api_key
self.misp_converter.api_url = server_details.baseurl
self.misp_converter.tag = server_details.name
# TODO check if it is a pull server
user = self.event_controller.user_broker.get_by_id(
self.get_user().identifier)
self.misp_converter.user = user
recent_events = self.misp_converter.get_index()
local_events = self.event_controller.get_event_by_uuids(
recent_events.keys())
items_to_remove = list()
# find the ones do not need to be updated
for local_event in local_events:
values = recent_events[local_event.uuid]
if values[1] <= local_event.last_publish_date:
items_to_remove.append(local_event.uuid)
for item_to_remove in items_to_remove:
del recent_events[item_to_remove]
for value in recent_events.itervalues():
self.process_controller.create_new_process(ProcessType.PULL,
value[0], user,
server_details)
# fetch one event from misp
# misp_event_xml = self.misp_converter.get_xml_event(value[0])
# make insert/merge
# try:
# self.ins_merg_event(server_details, misp_event_xml)
# except BrokerException as error:
# self.logger.error(error)
# TODO dump xml or log it in browser
return 'OK'
def push(self, server_details):
if server_details.type != 'MISP':
raise MISPAdapterException(
'Server {0} is not a MISP server'.format(
server_details.identifier))
self.misp_converter.api_key = server_details.user.api_key
self.misp_converter.api_url = server_details.baseurl
self.misp_converter.tag = server_details.name
# TODO check if it is a pull server
user = self.event_controller.user_broker.get_by_id(
self.get_user().identifier)
self.misp_converter.user = user
# everything is handled inside here
return self.misp_converter.filter_event_push(self, server_details)
def make_misp_xml(self, event, server_user):
flat_attribtues = self.ce1sus_to_misp.relations_controller.get_flat_attributes_for_event(
event)
result = list()
for flat_attribtue in flat_attribtues:
if self.is_item_viewable(
event, flat_attribtue,
server_user) and flat_attribtue.properties.is_shareable:
result.append(flat_attribtue)
references = list()
for report in event.reports:
for reference in report.references:
if self.is_item_viewable(
event, reference,
server_user) and reference.properties.is_shareable:
references.append(reference)
result = self.ce1sus_to_misp.create_event_xml(event, result,
references)
return result
@cherrypy.expose
@cherrypy.tools.allow(methods=['GET', 'POST'])
def events(self, *vpath, **params):
headers = cherrypy.request.headers
authkey = headers.get('Authorization')
# reset key
headers['key'] = authkey
self.login_handler.login(headers=headers)
# does not handle sessions well :P
try:
server_user = self.event_controller.user_broker.get_user_by_api_key(
authkey)
except BrokerException as error:
self.logger.error(error)
raise cherrypy.HTTPError(403)
rawbody = ''
path = vpath
method = cherrypy.request.method
cl = headers.get('Content-Length', None)
if cl:
rawbody = cherrypy.request.body.read(int(cl))
content_type = headers.get('Content-Type', 'application/json')
if content_type == 'application/json':
body = json.loads(rawbody)
elif content_type == 'application/xml':
body = rawbody
else:
raise
# check aut else a 405 exception
if len(path) > 0:
detected = False
if path[0] == 'filterEventIdsForPush':
if method == 'POST':
return_message = self.perform_push(body)
detected = True
elif path[0] == 'index':
if method == 'GET':
cherrypy.response.headers[
'Content-Type'] = 'application/xml; charset=UTF-8'
return_message = self.make_index(server_user)
detected = True
if not detected:
try:
event_id = int(path[0])
event = self.event_controller.get_event_by_id(event_id)
self.is_event_viewable(event, server_user)
# TODO check if user can view event
# convert event to misp event
cherrypy.response.headers[
'Content-Type'] = 'application/xml; charset=UTF-8'
misp_event = self.make_misp_xml(event, server_user)
return_message = misp_event
except ValueError:
raise cherrypy.HTTPError(404)
else:
# it is an insert of an event
if method == 'POST':
cherrypy.response.headers[
'Content-Type'] = 'application/xml; charset=UTF-8'
return_message = self.pushed_event(body, server_user)
self.logout_handler.logout()
return return_message
def ins_merg_event(self,
server_details,
xml_string,
server_user=None,
user=None):
if server_details.type == 'MISP':
if user:
pass
else:
user = self.event_controller.user_broker.get_by_id(
self.get_user().identifier)
self.misp_converter.api_key = server_details.user.api_key
self.misp_converter.api_url = server_details.baseurl
self.misp_converter.tag = server_details.name
self.misp_converter.user = user
merged_event = None
try:
event_uuid = self.misp_converter.get_uuid_from_event_xml(
xml_string)
try:
event = self.misp_converter.get_event_from_xml(
xml_string, None)
self.logger.info('Received Event {0}'.format(event.uuid))
event.properties.is_validated = False
self.event_controller.insert_event(user, event, True, True)
except ControllerIntegrityException as error:
local_event = self.event_controller.get_event_by_uuid(
event_uuid)
event = self.misp_converter.get_event_from_xml(
xml_string, local_event)
# merge event with existing event
if self.is_event_viewable(local_event, server_user):
event_permissions = self.get_event_user_permissions(
event, user)
try:
merged_event = self.merger.merge_event(
local_event, event, user, event_permissions)
except MergingException:
raise MISPAdapterException405()
# raise cherrypy.HTTPError(405)
else:
# TODO log the changes
self.logger.warning(
'user {0} tried to change event {1} but does not have the right to see it'
.format(user.username, event.identifer))
if merged_event:
self.logger.info('Received Event {0} updates'.format(
merged_event.uuid))
self.event_controller.update_event(
user, merged_event, True, True)
event = merged_event
else:
self.logger.info(
'Received Event {0} did not need to update as it is up to date'
.format(event.uuid))
# Log errors however
self.event_controller.event_broker.do_commit()
return self.make_misp_xml(event, server_user)
except BrokerException as error:
self.logger.error('Received a MISP Event which caused errors')
self.logger.error(error)
# TODO Dump xml
raise MISPAdapterException500()
# raise cherrypy.HTTPError(500)
else:
raise MISPAdapterException409('Server is not a MISP Server')
# raise cherrypy.HTTPError(409, 'Server is not a MISP Server')
def pushed_event(self, xml_string, server_user):
# instantiate misp converter
user = self.get_user()
try:
server_details = self.server_broker.get_server_by_user_id(
user.identifier)
if server_details.type != 'MISP':
raise cherrypy.HTTPError(409, 'Server is not a MISP Server')
# check if it is a misp server else raise
return self.ins_merg_event(server_details, xml_string, server_user)
except BrokerException as error:
self.logger.error(error)
raise cherrypy.HTTPError(404)
except MISPAdapterException405 as error:
raise cherrypy.HTTPError(405)
except MISPAdapterException409 as error:
raise cherrypy.HTTPError(409)
except MISPAdapterException500 as error:
raise cherrypy.HTTPError(500)
def perform_push(self, send_events):
incomming_events = dict()
for send_event in send_events:
uuid = send_event['Event']['uuid']
incomming_events[uuid] = send_event['Event']['timestamp']
remove = list()
# find all events matching the uuids
local_events = self.event_controller.get_event_by_uuids(
incomming_events.keys())
for local_event in local_events:
# check if the local events were not modified
date = incomming_events[local_event.uuid]
datetime_from_string = datetime.utcfromtimestamp(int(date))
if local_event.last_publish_date >= datetime_from_string:
# remove the ones which are either new or
remove.append(local_event.uuid)
# check if the local event is not locked -> does not exist in ours
# TODO: implement a lock?
result = list()
# create array of uuids
for key in incomming_events.iterkeys():
if key not in remove:
result.append(key)
cherrypy.response.headers[
'Content-Type'] = 'application/json; charset=UTF-8'
return json.dumps(result)
@cherrypy.expose
@cherrypy.tools.allow(methods=['GET'])
@require()
def event(self, *vpath, **params):
if len(vpath) > 0:
uuid_string = vpath[0]
# check if it is a uuid
# check the mode
make_file = params.get('file', None)
if make_file == '':
cherrypy.response.headers[
'Content-Type'] = 'application/x-download'
cherrypy.response.headers[
"Content-Disposition"] = 'attachment; filename=Event_{0}_MISP.xml'.format(
uuid_string)
else:
cherrypy.response.headers['Content-Type'] = 'text/xml'
try:
UUID(uuid_string, version=4)
except ValueError as error:
print error
raise HTTPError(
400,
'The provided uuid "{0}" is not valid'.format(uuid_string))
try:
event = self.event_controller.get_event_by_uuid(uuid_string)
return self.make_misp_xml(event, self.get_user())
except ControllerNothingFoundException as error:
raise HTTPError(404, '{0}'.format(error.message))
else:
raise HTTPError(400, 'Cannot be called without a uuid')