def __init__(self, config):
self.config = config
self.session_manager = SessionManager(config)
directconnection = self.session_manager.connector.get_direct_session()
self.process_controller = ProcessController(config, directconnection)
self.server_controller = SyncServerController(config, directconnection)
self.misp_converter = MispConverter(config, None, None, None,
directconnection)
dump = config.get('MISPAdapter', 'dump', False)
file_loc = config.get('MISPAdapter', 'file', None)
self.misp_converter.dump = dump
self.misp_converter.file_location = file_loc
self.misp_ctrl = MISPAdapter(config, directconnection)
user_uuid = config.get('ce1sus', 'maintenaceuseruuid', None)
self.user_controller = UserController(config, directconnection)
self.event_controller = EventController(config, directconnection)
self.group_broker = self.user_controller.broker_factory(GroupBroker)
self.mail_controller = MailController(config, directconnection)
self.ce1sus_adapter = Ce1susAdapter(config, directconnection)
if None:
raise SchedulerException(
'maintenaceuseruuid was not defined in config')
try:
self.user = self.user_controller.get_user_by_uuid(user_uuid)
except ControllerNothingFoundException:
raise SchedulerException(
'Cannot find maintenance user with uuid {0}'.format(user_uuid))
except ControllerException as error:
raise SchedulerException(error)
class StixMapper(BaseController):
def __init__(self, config, session=None):
BaseController.__init__(self, config, session)
self.config = config
ce1sus_url = config.get('ce1sus', 'baseurl', None)
self.event_controller = EventController(config, session)
self.stix_ce1sus_mapper = StixCelsusMapper(config, session)
self.ce1sus_stix_mapper = Ce1susStixMapper(config, session)
if ce1sus_url:
# Set the namespaces
cybox.utils.idgen.set_id_namespace(Namespace(ce1sus_url, 'ce1sus'))
stix.utils.idgen.set_id_namespace({ce1sus_url: 'ce1sus'})
else:
raise StixMapperException(u'the base url was not specified in configuration')
def map_stix_package(self, stix_package, user, make_insert=True, ignore_id=False):
event = self.stix_ce1sus_mapper.create_event(stix_package, user, ignore_id)
set_extended_logging(event, user, user.group)
if make_insert:
self.event_controller.insert_event(user, event, True, False)
return event
def map_ce1sus_event(self, event, user):
stix_package = self.ce1sus_stix_mapper.create_stix(event, user)
return stix_package.to_xml()
def __init__(self, config):
self.config = config
self.session_manager = SessionManager(config)
directconnection = self.session_manager.connector.get_direct_session()
self.relation_controller = RelationController(config, directconnection)
self.attribute_definition_controller = AttributeDefinitionController(
config, directconnection)
self.object_definition_controller = ObjectDefinitionController(
config, directconnection)
self.event_controller = EventController(config, directconnection)
self.user_controller = UserController(config, directconnection)
self.reference_controller = ReferencesController(
config, directconnection)
self.conditions_controller = ConditionController(
config, directconnection)
self.verbose = False
# set maintenance user
user_uuid = config.get('ce1sus', 'maintenaceuseruuid', None)
if None:
raise MaintenanceException(
'maintenaceuseruuid was not defined in config')
try:
self.user = self.user_controller.get_user_by_uuid(user_uuid)
except ControllerNothingFoundException:
raise MaintenanceException(
'Cannot find maintenance user with uuid {0}'.format(user_uuid))
except ControllerException as error:
raise MaintenanceException(error)
def __init__(self, config, session=None):
self.server_details = None
self.proxies = {}
self.verify_ssl = False
self.ssl_cert = None
self.session = requests.session()
self.assembler = Assembler(config, session)
self.event_controller = EventController(config, session)
self.process_controller = ProcessController(config, session)
def __init__(self, config):
"""
Creator
:param config: The configuration for this module
:type config: Configuration
:returns: BaseView
"""
self.config = config
self.__logger = Log(config)
self.event_controller = EventController(config)
self.user_controller = UserController(config)
def __init__(self, config, session=None):
BaseController.__init__(self, config, session)
self.config = config
ce1sus_url = config.get('ce1sus', 'baseurl', None)
self.event_controller = EventController(config, session)
self.stix_ce1sus_mapper = StixCelsusMapper(config, session)
self.ce1sus_stix_mapper = Ce1susStixMapper(config, session)
if ce1sus_url:
# Set the namespaces
cybox.utils.idgen.set_id_namespace(Namespace(ce1sus_url, 'ce1sus'))
stix.utils.idgen.set_id_namespace({ce1sus_url: 'ce1sus'})
else:
raise StixMapperException(u'the base url was not specified in configuration')
def __init__(self, config, session=None):
BaseView.__init__(self, config)
self.event_controller = EventController(config, session)
self.server_broker = self.event_controller.broker_factory(
SyncServerBroker)
self.ce1sus_to_misp = Ce1susMISP(config, session)
self.login_handler = LoginHandler(config)
self.logout_handler = LogoutHandler(config)
self.misp_converter = MispConverter(config, None, None, None, session)
dump = config.get('MISPAdapter', 'dump', False)
file_loc = config.get('MISPAdapter', 'file', None)
self.misp_converter.dump = dump
self.misp_converter.file_location = file_loc
self.merger = Merger(config, session)
self.process_controller = ProcessController(config, session)
def __init__(self, config, dumps_folder):
self.config = config
self.dumps_folder = dumps_folder
self.logger = Log(config).get_logger('Main')
connector = SessionManager(config).connector
directconnection = connector.get_direct_session()
self.session = directconnection
self.notmapped = open(self.dumps_folder + '/notmapped.txt', 'w')
self.event_controller = EventController(config, directconnection)
self.attr_def_controller = AttributeDefinitionController(config, directconnection)
self.obj_def_controller = ObjectDefinitionController(config, directconnection)
def_con = ConditionController(config, directconnection)
self.conditions = def_con.get_all_conditions()
def_con = ReferencesController(config, directconnection)
self.ressources = def_con.get_reference_definitions_all()
def_con = IndicatorController(config, directconnection)
self.types = def_con.get_all_types()
self.user_controller = UserController(config, directconnection)
self.group_controller = GroupController(config, directconnection)
self.users = dict()
self.groups = dict()
self.seen_attributes = dict()
self.seen_attr_defs = dict()
self.seen_obj_defs = dict()
def __init__(self, config, session=None):
BaseController.__init__(self, config, session)
self.event_controller = EventController(config, session)
self.relations_controller = RelationController(config, session)
self.events_controller = EventsController(config, session)
self.rel_def_broker = self.broker_factory(ReferenceDefintionsBroker)
self.report_broker = self.broker_factory(ReportBroker)
def __init__(self, config, session=None):
BaseController.__init__(self, config, session)
self.observable_controller = ObservableController(config, session)
self.event_controller = EventController(config, session)
self.condition_broker = self.broker_factory(ConditionBroker)
self.reference_definiton_broker = self.broker_factory(
ReferenceDefintionsBroker)
self.handler_broker = self.broker_factory(AttributeHandlerBroker)
self.value_type_broker = self.broker_factory(AttributeTypeBroker)
self.condition_broker = self.broker_factory(ConditionBroker)
self.references_broker = self.broker_factory(ReferencesBroker)
self.salt = self.config.get('ce1sus', 'salt', None)
class MISPAdapter(BaseView):
def __init__(self, config, session=None):
BaseView.__init__(self, config)
self.event_controller = EventController(config, session)
self.server_broker = self.event_controller.broker_factory(
SyncServerBroker)
self.ce1sus_to_misp = Ce1susMISP(config, session)
self.login_handler = LoginHandler(config)
self.logout_handler = LogoutHandler(config)
self.misp_converter = MispConverter(config, None, None, None, session)
dump = config.get('MISPAdapter', 'dump', False)
file_loc = config.get('MISPAdapter', 'file', None)
self.misp_converter.dump = dump
self.misp_converter.file_location = file_loc
self.merger = Merger(config, session)
self.process_controller = ProcessController(config, session)
@cherrypy.expose
@cherrypy.tools.allow(methods=['POST'])
@require()
def upload_xml(self, *vpath, **params):
try:
input_json = self.get_json()
filename = input_json['name']
data = input_json['data']['data']
xml_string = base64.b64decode(data)
complete = params.get('complete', False)
inflated = params.get('inflated', False)
user = self.get_user()
user = self.user_controller.get_user_by_username(user.username)
if 'XML' in filename or 'xml' in filename:
self.logger.info(
'Starting to import xml form file {0}'.format(filename))
self.misp_converter.user = user
event_uuid = self.misp_converter.get_uuid_from_event_xml(
xml_string)
try:
event = self.misp_converter.get_event_from_xml(
xml_string, None)
self.logger.info('Received Event {0}'.format(event.uuid))
self.event_controller.insert_event(user, event, True, True)
except ControllerIntegrityException as error:
local_event = self.event_controller.get_event_by_uuid(
event_uuid)
event = self.misp_converter.get_event_from_xml(
xml_string, local_event)
# merge event with existing event
if self.is_event_viewable(local_event, user):
event_permissions = self.get_event_user_permissions(
event, user)
try:
merged_event = self.merger.merge_event(
local_event, event, user, event_permissions)
except MergingException:
raise MISPAdapterException405()
# raise cherrypy.HTTPError(405)
else:
# TODO log the changes
self.logger.warning(
'user {0} tried to change event {1} but does not have the right to see it'
.format(user.username, event.identifer))
raise HTTPError(403, 'Not authorized')
if merged_event:
self.logger.info('Received Event {0} updates'.format(
merged_event.uuid))
self.event_controller.update_event(
user, merged_event, True, True)
event = merged_event
else:
self.logger.info(
'Received Event {0} did not need to update as it is up to date'
.format(event.uuid))
# Log errors however
self.event_controller.event_broker.do_commit()
event_permissions = self.event_controller.get_event_user_permissions(
event, user)
cherrypy.response.headers[
'Content-Type'] = 'application/json; charset=UTF-8'
return event.to_dict(complete, inflated, event_permissions,
user)
else:
raise HTTPError(409, 'File does not end in xml or XML')
except MispConverterException as error:
self.logger.error(error)
raise HTTPError(409, 'File is not a MISP XML')
except ControllerException as error:
self.logger.error(error)
raise HTTPError(400, '{0}'.format(error.message))
except Exception as error:
self.logger.critical(error)
raise HTTPError(500, '{0}'.format(error.message))
@cherrypy.expose
@cherrypy.tools.allow(methods=['GET'])
def shadow_attributes(self, *vpath, **params):
# this is called from the misp server to see it his know events have new proposals
# TODO: Proposal for misp
raise cherrypy.HTTPError(404)
def get_all_events(self, server_user):
events = self.event_controller.get_all()
result = list()
for event in events:
if event.properties.is_validated:
if self.is_event_viewable(event, server_user):
result.append(event)
else:
if event.originating_group.identifier == server_user.group.identifier:
result.append(event)
return result
def make_index(self, server_user):
result = self.get_all_events(server_user)
return self.ce1sus_to_misp.make_index(result)
def pull(self, server_details):
# use the logged in user for these request
if server_details.type != 'MISP':
raise MISPAdapterException(
'Server {0} is not a MISP server'.format(
server_details.identifier))
self.misp_converter.api_key = server_details.user.api_key
self.misp_converter.api_url = server_details.baseurl
self.misp_converter.tag = server_details.name
# TODO check if it is a pull server
user = self.event_controller.user_broker.get_by_id(
self.get_user().identifier)
self.misp_converter.user = user
recent_events = self.misp_converter.get_index()
local_events = self.event_controller.get_event_by_uuids(
recent_events.keys())
items_to_remove = list()
# find the ones do not need to be updated
for local_event in local_events:
values = recent_events[local_event.uuid]
if values[1] <= local_event.last_publish_date:
items_to_remove.append(local_event.uuid)
for item_to_remove in items_to_remove:
del recent_events[item_to_remove]
for value in recent_events.itervalues():
self.process_controller.create_new_process(ProcessType.PULL,
value[0], user,
server_details)
# fetch one event from misp
# misp_event_xml = self.misp_converter.get_xml_event(value[0])
# make insert/merge
# try:
# self.ins_merg_event(server_details, misp_event_xml)
# except BrokerException as error:
# self.logger.error(error)
# TODO dump xml or log it in browser
return 'OK'
def push(self, server_details):
if server_details.type != 'MISP':
raise MISPAdapterException(
'Server {0} is not a MISP server'.format(
server_details.identifier))
self.misp_converter.api_key = server_details.user.api_key
self.misp_converter.api_url = server_details.baseurl
self.misp_converter.tag = server_details.name
# TODO check if it is a pull server
user = self.event_controller.user_broker.get_by_id(
self.get_user().identifier)
self.misp_converter.user = user
# everything is handled inside here
return self.misp_converter.filter_event_push(self, server_details)
def make_misp_xml(self, event, server_user):
flat_attribtues = self.ce1sus_to_misp.relations_controller.get_flat_attributes_for_event(
event)
result = list()
for flat_attribtue in flat_attribtues:
if self.is_item_viewable(
event, flat_attribtue,
server_user) and flat_attribtue.properties.is_shareable:
result.append(flat_attribtue)
references = list()
for report in event.reports:
for reference in report.references:
if self.is_item_viewable(
event, reference,
server_user) and reference.properties.is_shareable:
references.append(reference)
result = self.ce1sus_to_misp.create_event_xml(event, result,
references)
return result
@cherrypy.expose
@cherrypy.tools.allow(methods=['GET', 'POST'])
def events(self, *vpath, **params):
headers = cherrypy.request.headers
authkey = headers.get('Authorization')
# reset key
headers['key'] = authkey
self.login_handler.login(headers=headers)
# does not handle sessions well :P
try:
server_user = self.event_controller.user_broker.get_user_by_api_key(
authkey)
except BrokerException as error:
self.logger.error(error)
raise cherrypy.HTTPError(403)
rawbody = ''
path = vpath
method = cherrypy.request.method
cl = headers.get('Content-Length', None)
if cl:
rawbody = cherrypy.request.body.read(int(cl))
content_type = headers.get('Content-Type', 'application/json')
if content_type == 'application/json':
body = json.loads(rawbody)
elif content_type == 'application/xml':
body = rawbody
else:
raise
# check aut else a 405 exception
if len(path) > 0:
detected = False
if path[0] == 'filterEventIdsForPush':
if method == 'POST':
return_message = self.perform_push(body)
detected = True
elif path[0] == 'index':
if method == 'GET':
cherrypy.response.headers[
'Content-Type'] = 'application/xml; charset=UTF-8'
return_message = self.make_index(server_user)
detected = True
if not detected:
try:
event_id = int(path[0])
event = self.event_controller.get_event_by_id(event_id)
self.is_event_viewable(event, server_user)
# TODO check if user can view event
# convert event to misp event
cherrypy.response.headers[
'Content-Type'] = 'application/xml; charset=UTF-8'
misp_event = self.make_misp_xml(event, server_user)
return_message = misp_event
except ValueError:
raise cherrypy.HTTPError(404)
else:
# it is an insert of an event
if method == 'POST':
cherrypy.response.headers[
'Content-Type'] = 'application/xml; charset=UTF-8'
return_message = self.pushed_event(body, server_user)
self.logout_handler.logout()
return return_message
def ins_merg_event(self,
server_details,
xml_string,
server_user=None,
user=None):
if server_details.type == 'MISP':
if user:
pass
else:
user = self.event_controller.user_broker.get_by_id(
self.get_user().identifier)
self.misp_converter.api_key = server_details.user.api_key
self.misp_converter.api_url = server_details.baseurl
self.misp_converter.tag = server_details.name
self.misp_converter.user = user
merged_event = None
try:
event_uuid = self.misp_converter.get_uuid_from_event_xml(
xml_string)
try:
event = self.misp_converter.get_event_from_xml(
xml_string, None)
self.logger.info('Received Event {0}'.format(event.uuid))
event.properties.is_validated = False
self.event_controller.insert_event(user, event, True, True)
except ControllerIntegrityException as error:
local_event = self.event_controller.get_event_by_uuid(
event_uuid)
event = self.misp_converter.get_event_from_xml(
xml_string, local_event)
# merge event with existing event
if self.is_event_viewable(local_event, server_user):
event_permissions = self.get_event_user_permissions(
event, user)
try:
merged_event = self.merger.merge_event(
local_event, event, user, event_permissions)
except MergingException:
raise MISPAdapterException405()
# raise cherrypy.HTTPError(405)
else:
# TODO log the changes
self.logger.warning(
'user {0} tried to change event {1} but does not have the right to see it'
.format(user.username, event.identifer))
if merged_event:
self.logger.info('Received Event {0} updates'.format(
merged_event.uuid))
self.event_controller.update_event(
user, merged_event, True, True)
event = merged_event
else:
self.logger.info(
'Received Event {0} did not need to update as it is up to date'
.format(event.uuid))
# Log errors however
self.event_controller.event_broker.do_commit()
return self.make_misp_xml(event, server_user)
except BrokerException as error:
self.logger.error('Received a MISP Event which caused errors')
self.logger.error(error)
# TODO Dump xml
raise MISPAdapterException500()
# raise cherrypy.HTTPError(500)
else:
raise MISPAdapterException409('Server is not a MISP Server')
# raise cherrypy.HTTPError(409, 'Server is not a MISP Server')
def pushed_event(self, xml_string, server_user):
# instantiate misp converter
user = self.get_user()
try:
server_details = self.server_broker.get_server_by_user_id(
user.identifier)
if server_details.type != 'MISP':
raise cherrypy.HTTPError(409, 'Server is not a MISP Server')
# check if it is a misp server else raise
return self.ins_merg_event(server_details, xml_string, server_user)
except BrokerException as error:
self.logger.error(error)
raise cherrypy.HTTPError(404)
except MISPAdapterException405 as error:
raise cherrypy.HTTPError(405)
except MISPAdapterException409 as error:
raise cherrypy.HTTPError(409)
except MISPAdapterException500 as error:
raise cherrypy.HTTPError(500)
def perform_push(self, send_events):
incomming_events = dict()
for send_event in send_events:
uuid = send_event['Event']['uuid']
incomming_events[uuid] = send_event['Event']['timestamp']
remove = list()
# find all events matching the uuids
local_events = self.event_controller.get_event_by_uuids(
incomming_events.keys())
for local_event in local_events:
# check if the local events were not modified
date = incomming_events[local_event.uuid]
datetime_from_string = datetime.utcfromtimestamp(int(date))
if local_event.last_publish_date >= datetime_from_string:
# remove the ones which are either new or
remove.append(local_event.uuid)
# check if the local event is not locked -> does not exist in ours
# TODO: implement a lock?
result = list()
# create array of uuids
for key in incomming_events.iterkeys():
if key not in remove:
result.append(key)
cherrypy.response.headers[
'Content-Type'] = 'application/json; charset=UTF-8'
return json.dumps(result)
@cherrypy.expose
@cherrypy.tools.allow(methods=['GET'])
@require()
def event(self, *vpath, **params):
if len(vpath) > 0:
uuid_string = vpath[0]
# check if it is a uuid
# check the mode
make_file = params.get('file', None)
if make_file == '':
cherrypy.response.headers[
'Content-Type'] = 'application/x-download'
cherrypy.response.headers[
"Content-Disposition"] = 'attachment; filename=Event_{0}_MISP.xml'.format(
uuid_string)
else:
cherrypy.response.headers['Content-Type'] = 'text/xml'
try:
UUID(uuid_string, version=4)
except ValueError as error:
print error
raise HTTPError(
400,
'The provided uuid "{0}" is not valid'.format(uuid_string))
try:
event = self.event_controller.get_event_by_uuid(uuid_string)
return self.make_misp_xml(event, self.get_user())
except ControllerNothingFoundException as error:
raise HTTPError(404, '{0}'.format(error.message))
else:
raise HTTPError(400, 'Cannot be called without a uuid')
class Ce1susAdapter(BaseController):
def __init__(self, config, session=None):
self.server_details = None
self.proxies = {}
self.verify_ssl = False
self.ssl_cert = None
self.session = requests.session()
self.assembler = Assembler(config, session)
self.event_controller = EventController(config, session)
self.process_controller = ProcessController(config, session)
@property
def apiUrl(self):
return '{0}/REST/0.3.0'.format(self.server_details.baseurl)
@property
def apiKey(self):
return self.server_details.user.api_key
def __set_complete_inflated(self, url, complete=False, inflated=False):
if complete and not inflated:
url = '{0}?complete=true'.format(url)
if not complete and inflated:
url = '{0}?inflated=true'.format(url)
if complete and inflated:
url = '{0}?complete=true&inflated=true'.format(url)
return url
def __extract_message(self, error):
reason = error.message
message = error.response.text
code = error.response.status_code
# "<p>An event with uuid "54f63b0f-0c98-4e74-ab95-60c718689696" already exists</p>
try:
pos = message.index('<p>') + 3
message = message[pos:]
pos = message.index('</p>')
message = message[:pos]
except ValueError:
# In case the message is not parsable
pass
return code, reason, message
def __handle_exception(self, request):
try:
request.raise_for_status()
except requests.exceptions.HTTPError as error:
code, reason, message = self.__extract_message(error)
message = u'{0} ({1})'.format(reason, message)
if code == 403:
raise Ce1susAdapterForbiddenException(message)
elif code == 404:
raise Ce1susAdapterNothingFoundException(message)
else:
raise Ce1susAdapterException(message)
def __request(self, path, method, data=None, extra_headers=None):
try:
url = '{0}/{1}'.format(self.apiUrl, path)
headers = {
'Content-Type': 'application/json; charset=utf-8',
'User-Agent': 'Ce1sus API server client {0}'.format(APP_REL),
'key': self.apiKey
}
if extra_headers:
for key, value in extra_headers.items():
headers[key] = value
if method == 'GET':
request = self.session.get(url,
headers=headers,
proxies=self.proxies,
verify=self.verify_ssl,
cert=self.ssl_cert,
cookies=self.session.cookies)
elif method == 'PUT':
request = self.session.put(url,
json.dumps(data),
headers=headers,
proxies=self.proxies,
verify=self.verify_ssl,
cert=self.ssl_cert,
cookies=self.session.cookies)
elif method == 'DELETE':
request = self.session.delete(url,
headers=headers,
proxies=self.proxies,
verify=self.verify_ssl,
cert=self.ssl_cert,
cookies=self.session.cookies)
elif method == 'POST':
request = self.session.post(url,
json.dumps(data),
headers=headers,
proxies=self.proxies,
verify=self.verify_ssl,
cert=self.ssl_cert,
cookies=self.session.cookies)
else:
raise UnkownMethodException(
u'Mehtod {0} is not specified can only be GET,POST,PUT or DELETE'
)
if request.status_code == requests.codes.ok:
return json.loads(request.text)
else:
self.__handle_exception(request)
except requests.exceptions.RequestException as error:
raise Ce1susAdapterException(error)
except requests.ConnectionError as error:
raise Ce1susAdapterConnectionException('{0}'.format(error.message))
def login(self):
text = self.__request('/login', 'POST', None)
return text
def logout(self):
text = self.__request(
'/logout',
'GET',
)
if text == 'User logged out':
return True
else:
return False
def get_event_by_uuid(self,
uuid,
complete=False,
inflated=False,
poponly=True,
json=False):
url = '/event/{0}'.format(uuid)
url = self.__set_complete_inflated(url, complete, inflated)
json = self.__request(url, 'GET', None)
if json:
return json
else:
event = self.assembler.assemble_event(json,
self.server_details.user,
False, True, poponly)
return event
def insert_event(self, event, complete=False, inflated=False):
url = '/event'
url = self.__set_complete_inflated(url, complete, inflated)
event_permissions = self.event_controller.get_event_user_permissions(
event, self.server_details.user)
json = self.__request(url,
'POST',
data=event.to_dict(True, True, event_permissions,
self.server_details.user))
owner = is_event_owner(event, self.server_details.user)
event = self.assembler.update_event(event, json,
self.server_details.user, owner,
True)
return event
def update_event(self, event, complete=False, inflated=False):
url = '/event/{0}'.format(event.uuid)
url = self.__set_complete_inflated(url, complete, inflated)
event_permissions = self.event_controller.get_event_user_permissions(
event, self.server_details.user)
json = self.__request(url,
'PUT',
data=event.to_dict(True, True, event_permissions,
self.server_details.user))
owner = is_event_owner(event, self.server_details.user)
event = self.assembler.update_event(event, json,
self.server_details.user, owner,
True)
return event
def get_index(self, server_details=None):
url = '/events'
if server_details:
self.server_details = server_details
url = self.__set_complete_inflated(url, False, False)
json = self.__request(url, 'GET', None)
events_json = json.get('data', list())
result = list()
for event_json in events_json:
event = self.assembler.assemble_event(event_json,
self.server_details.user,
False, True, True)
result.append(event)
return result
def push(self, server_details):
try:
self.server_details = server_details
self.login()
user_events = self.event_controller.get_all_for_user(
server_details.user)
# get the remote ones
rem_events = self.get_index(server_details)
rem_events_dict = dict()
for rem_event in rem_events:
rem_events_dict[rem_event.uuid] = rem_event
uuids_to_push = list()
for event in user_events:
rem_event = rem_events_dict.get(event.uuid, None)
if rem_event:
if event.last_publish_date and rem_event.last_publish_date:
if event.last_publish_date > rem_event.last_publish_date:
uuids_to_push.append(event.uuid)
else:
uuids_to_push.append(event.uuid)
# pass
for uuid_to_push in uuids_to_push:
self.process_controller.create_new_process(
ProcessType.PUSH, uuid_to_push, self.server_details.user,
server_details, True)
except Ce1susAdapterException as error:
self.logout()
raise Ce1susAdapterException(error)
self.logout()
def pull(self, server_details):
try:
self.server_details = server_details
self.login()
events = self.get_index(server_details)
event_uuids = dict()
for event in events:
event_uuids[event.uuid] = event
local_events = self.event_controller.get_event_by_uuids(
event_uuids.keys())
items_to_remove = list()
for local_event in local_events:
rem_event = event_uuids[local_event.uuid]
if local_event.last_publish_date and rem_event.last_publish_date:
if rem_event.last_publish_date <= local_event.last_publish_date:
items_to_remove.append(local_event.uuid)
else:
items_to_remove.append(local_event.uuid)
for item_to_remove in items_to_remove:
del event_uuids[item_to_remove]
for rem_event in event_uuids.itervalues():
self.process_controller.create_new_process(
ProcessType.PULL, rem_event.uuid, server_details.user,
server_details, True)
self.logout()
return 'OK'
except Ce1susAdapterException as error:
self.logout()
raise Ce1susAdapterException(error)
class BaseView(object):
def __init__(self, config):
"""
Creator
:param config: The configuration for this module
:type config: Configuration
:returns: BaseView
"""
self.config = config
self.__logger = Log(config)
self.event_controller = EventController(config)
self.user_controller = UserController(config)
def get_json(self):
json = {}
cl = cherrypy.request.headers.get('Content-Length', None)
if cl:
try:
rawbody = cherrypy.request.body.read(int(cl))
json = loads(rawbody)
except TypeError:
# wonder what's wrong restangular?!?
counter = 0
rawbody = ''
while True:
rawbody += cherrypy.request.body.readline(counter)
try:
json = loads(rawbody)
break
except ValueError:
counter += 1
return json
@property
def logger(self):
return self.__logger.get_logger(self.__class__.__name__)
def _create_session(self):
"""
creates a session in cherrypy
"""
session = self._get_session()
session.regenerate()
self.logger.debug('Created a session')
def _put_to_session(self, key, value):
"""
puts/sets a key value pair to the session
:param key: The key for the value
:type key: object
:param value: The value for the key
:type value: object
"""
session = self._get_session()
session[key] = value
self.logger.debug('Set session value {0} for key {1}'.format(
value, key))
def __is_session_key(self, key):
"""
Checks if the key is existing the session, else raises a SessionNotFoundException
:param key: The key for the value
:type key: object
"""
session = self._get_session()
if key not in session.keys():
self.logger.debug('Key {0} is not defined in session'.format(key))
raise SessionNotFoundException(
'Key {0} was not defined in session'.format(key))
def _get_from_session(self, key, default_value=None):
"""
Get a variable by key from the session
Note: The variable stays in the session
:param key: The key for the value
:type key: object
:param value: The value for the key
:type value: object
:returns: object
"""
session = self._get_session()
value = session.get(key, default_value)
self.logger.debug('Returned session value "{0}" for key "{1}"'.format(
value, key))
return value
def _pull_from_session(self, key, default_value=None):
"""
Pulls a variable by key from the session
Note: The variable is removed from the session
:param key: The key for the value
:type key: object
:param value: The value for the key
:type value: object
:returns: object
"""
session = self._get_session()
value = session.pop(key, default_value)
self.logger.debug(
'Returned session value "{0}" for key "{1}" and removed it'.format(
value, key))
return value
def _destroy_session(self):
"""
Destroys a session
"""
try:
session = self._get_session()
session.clear()
session.delete()
# session.clean_up()
self.logger.debug('Session destroyed')
except CherryPyException as error:
self.logger.error(error)
def _get_session(self):
"""
Returns the session
"""
session = getattr(cherrypy, 'session')
self.logger.debug('Session returned')
return session
def user_authenticated(self):
username = self._get_from_session(SESSION_KEY, None)
return username
def get_user(self):
"""
Returns the user from the session
:returns: User
"""
user = self._get_from_session(SESSION_USER)
if user:
user = self.user_controller.get_user_by_username(user.username)
return user
def get_authorized_events_cache(self):
"""
Returns the authorized cached events
"""
try:
return self._get_from_session('_cp_events_cache', dict())
except AttributeError:
return dict()
def set_authorized_events_cache(self, cache):
try:
self._put_to_session('_cp_events_cache', cache)
except AttributeError:
pass
def check_if_admin(self):
user = self.get_user()
isadmin = is_user_priviledged(user)
if not isadmin:
raise cherrypy.HTTPError(
403, 'User {0} is not privileged'.format(user.username))
def check_if_admin_validate(self):
user = self.get_user()
if not user.permissions.validate:
raise cherrypy.HTTPError(
403, 'User {0} cannot validate events'.format(user.username))
def is_event_viewable(self, event, user=None):
# The same is in the mailer
if not user:
user = self.get_user()
if self.is_event_owner(event, user):
return True
else:
if user:
if user.group_id:
user_group = self.event_controller.group_broker.get_by_id(
user.group_id)
tlp_lvl = get_max_tlp(user_group)
if event.tlp_level_id >= tlp_lvl:
return True
else:
grp_ids = list()
for group in user_group.children:
grp_ids.append(group.identifier)
grp_ids.append(user.group_id)
for eventgroup in event.groups:
group = eventgroup.group
if group.identifier in grp_ids:
return True
return False
else:
return False
else:
return False
def check_if_event_is_viewable(self, event):
user = self.get_user()
if not self.is_event_viewable(event, user):
raise cherrypy.HTTPError(
403, 'User/group {0}/{1} cannot view this event'.format(
user.username, user.group.name))
def is_item_viewable(self, event, item, user=None):
if not user:
user = self.get_user()
if self.is_event_owner(event, user):
return True
else:
# check is the event is viewable then process to the iem
if self.is_event_viewable(event, user):
permissions = self.get_event_user_permissions(event, user)
if is_object_viewable(item, permissions, user):
return True
else:
return False
else:
return False
def check_item_is_viewable(self, event, item):
user = self.get_user()
result = self.is_item_viewable(event, item)
# if the result is still not set throw an error
log_msg = get_item_view_message(result, event.identifier,
item.identifier, user.username,
'can_view')
# update cache
self.logger.info(log_msg)
if result is None:
raise Exception(u'Unknown error occurred during event checks')
else:
return result
def check_if_event_is_modifiable(self, event):
self.check_permission(event, 'can_modify')
def check_if_event_is_deletable(self, event):
self.check_permission(event, 'can_delete')
def check_if_event_group_can_change(self, event):
self.check_permission(event, 'set_groups')
def check_permission(self, event, permission):
user = self.get_user()
result = self.is_user_allowed_to_perform(event, permission, user)
if not result:
raise cherrypy.HTTPError(
403,
'User/group {0}/{1} is not authorized perform action "{3}" on event {2}'
.format(user.username, user.group.name, event.identifier,
permission))
def check_if_owner(self, event):
user = self.get_user()
owner = is_event_owner(event, user)
if not owner:
raise cherrypy.HTTPError(
403, 'User/group {0}/{1} does not own event {2}'.format(
user.username, user.group.name, event.identifier))
def check_if_user_can_add(self, event):
user = self.get_user()
result = is_event_owner(event, user)
if not result:
result = self.is_user_allowed_to_perform(event, 'can_add', user)
if not result:
result = self.is_user_allowed_to_perform(
event, 'can_propose', user)
if not result:
raise cherrypy.HTTPError(
403, 'User {0}/{1} can not add contents to event {2}'.format(
user.username, user.group.name, event.identifier))
def is_user_allowed_to_perform(self, event, permission, user):
permissions = self.get_event_user_permissions(event, user)
if permissions:
result = getattr(permissions, permission)
# if the result is still not set throw an error
log_msg = get_view_message(result, event.identifier, user.username,
permission)
# update cache
self.logger.info(log_msg)
if result is None:
raise Exception(u'Unknown error occurred during event checks')
else:
return result
else:
return False
def is_user_priviledged(self, user):
self.logger.debug(u'Checking if user {0} is privileged'.format(
user.username))
result = is_user_priviledged(user)
if result:
self.logger.info(u'User {0} is privileged'.format(user.username))
else:
self.logger.info(u'User {0} is not privileged'.format(
user.username))
return result
def is_event_owner(self, event, user):
self.logger.debug(
u'Checking if user/group {0}/{1} owns event {2}'.format(
user.username, user.group.name, event.identifier))
result = is_event_owner(event, user)
if result:
self.logger.info(u'User/group {0}/{1} owns of event {2}'.format(
user.username, user.group.name, event.identifier))
else:
self.logger.info(
u'User/group {0}/{1} does not own of event {2}'.format(
user.username, user.group.name, event.identifier))
return result
def is_rest_insert(self, headers):
webinsert = headers.get('frontend', None)
if webinsert:
return False
else:
return True
def is_user_allowed_set_validate(self, event, old_instance, user, json):
properties = json.get('properties', None)
if properties:
permissions = self.get_event_user_permissions(event, user)
validated = properties.get('validated', None)
if validated is not None:
# validate is perhaps going to change
if permissions.can_validate:
return True
else:
# if there are changes deny them
if old_instance.properties.is_validated == validated:
# no change
return True
else:
self.logger.info(
u'User {0} has no right to validate elements of event {1}'
.format(user.username, event.identifier))
return False
return True
def is_user_allowed_set_share(self, event, old_instance, user, json):
properties = json.get('properties', None)
if properties:
shared = properties.get('shared', None)
if shared is not None:
if is_event_owner(event, user):
return True
else:
if old_instance.originating_group.identifier == user.group_id:
# User owns instance
return True
else:
if old_instance.properties.is_shareable == shared:
return True
else:
self.logger.info(
u'User/group {0}/{1} has no right to share elements of event {2}'
.format(user.username, user.group.name,
event.identifier))
return False
return True
def check_user_allowed_set_share(self, event, old_instance, user, json):
shared = self.is_user_allowed_set_share(event, old_instance, user,
json)
if not shared:
raise cherrypy.HTTPError(
403, 'User/group {0}/{1} cannot make this change'.format(
user.username, user.group.name))
def check_user_allowed_set_validate(self, event, old_instance, user, json):
validate = self.is_user_allowed_set_validate(event, old_instance, user,
json)
if not validate:
raise cherrypy.HTTPError(
403,
'User/group {0}/{1} can not validate elements of event {2}'.
format(user.username, user.group.name, event.identifier))
def is_if_user_can_set_validate_and_shared(self, event, old_instance, user,
json):
# TODO: Involve if user == partof the originator
validate = self.is_user_allowed_set_validate(event, old_instance, user,
json)
shared = self.is_user_allowed_set_validate(event, old_instance, user,
json)
return validate and shared
def check_if_user_can_set_validate_or_shared(self, event, old_instance,
user, json):
self.check_user_allowed_set_share(event, old_instance, user, json)
self.check_user_allowed_set_validate(event, old_instance, user, json)
def get_event_user_permissions(self, event, user):
cache = self.get_authorized_events_cache()
permissions = None
if event:
# check if cache
if cache:
permissions = cache.get(event.identifier, None)
# if None the event is not available inside the cache
if permissions:
return permissions
if permissions is None:
if self.is_event_owner(event, user):
permissions = EventPermissions('0')
permissions.set_all()
else:
# either there was no cache or the event was not inside the cache perform checks
permissions = self.event_controller.get_event_user_permissions(
event, user)
# put result in the cache if there is one
cache[event.identifier] = permissions
# perform checks
else:
permissions = EventPermissions('0')
self.set_authorized_events_cache(cache)
return permissions
def put_user_to_session(self, user):
cherrypy.request.login = user.username
self._put_to_session('_cp_username', user.username)
offline_user = self.__make_user_object(user)
self._put_to_session(SESSION_USER, offline_user)
def __make_user_object(self, user):
# TODO: make user offline
obj = GenObject()
obj.name = user.name
obj.username = user.username
obj.identifier = user.identifier
obj.email = user.email
obj.group_id = user.group_id
obj.activated = user.activated
obj.sirname = user.sirname
obj.permissions = UserRights(user.dbcode)
return obj
def __init__(self, config, session=None):
BaseController.__init__(self, config, session)
self.cybox_mapper = CyboxMapper(config, session)
self.event_controller = EventController(config, session)
self.observable_controller = ObservableController(config, session)
class StixCelsusMapper(BaseController):
def __init__(self, config, session=None):
BaseController.__init__(self, config, session)
self.cybox_mapper = CyboxMapper(config, session)
self.event_controller = EventController(config, session)
self.observable_controller = ObservableController(config, session)
def init(self):
groups = self.group_broker.get_all()
self.groups = make_dict_definitions(groups)
def __convert_info_soucre(self, info_source):
if info_source and info_source.identity:
identity = info_source.identity
identifier = None
if identity.id_:
identifier = extract_uuid(identity.id_)
# check if group exits
# Only based on the name as the id can vary
if not identity.name:
identity.name = 'Default'
group = self.groups.get(identity.name, None)
if group:
return group
else:
group = Group()
if identifier:
group.uuid = identifier
else:
# Unfortenately we must create our own
group.uuid = uuid4()
group.name = identity.name
group.description = u'Auto-generated from STIX'
group.tlp_lvl = 3
self.group_broker.insert(group, False)
self.groups[group.name] = group
return group
else:
return None
def __convert_sightings(self, sightings, user):
if sightings:
if sightings._inner:
result = list()
for sighting in sightings._inner:
ce1sus_sigthing = Sighting()
ce1sus_sigthing.created_at = sighting.timestamp
ce1sus_sigthing.timestamp_precision = sighting.timestamp_precision
ce1sus_sigthing.creator_group = self.__convert_info_soucre(
sighting.source)
ce1sus_sigthing.confidence = sighting.confidence
ce1sus_sigthing.description = sighting.description
set_extended_logging(ce1sus_sigthing, user, user.group)
result.append(ce1sus_sigthing)
return result
else:
return None
else:
return None
def __map_stix_package_header(self, stix_package, user):
self.init()
self.cybox_mapper.init()
event = Event()
event.uuid = extract_uuid(stix_package.id_)
# event.uuid = stix_package.id_[-36:]
stix_header = stix_package.stix_header
event.title = stix_header.title
if not event.title:
event.title = stix_header.short_description
if not event.title:
event.title = stix_header.description
if not event.title:
event.title = 'Generated via STIX - No title'
info_source = stix_header.information_source
event.description = stix_header.description
if info_source:
# set information source
set_extended_logging(event, user,
self.__convert_info_soucre(info_source))
if isinstance(stix_header.handling, Marking):
# set tlp markings if available
for specification in stix_header.handling._markings:
for structure in specification.marking_structures:
if isinstance(structure, TLPMarkingStructure):
# TODO: set desrcption with reference
event.tlp = structure.color.title()
if not event.tlp_level_id:
event.tlp = 'White'
# first and last seen
if stix_header.information_source:
# TODO review this
time = stix_header.information_source.time
event.created_at = time.produced_time.value
if time.received_time:
event.modified_on = time.received_time.value
else:
event.modified_on = event.created_at
event.first_seen = time.start_time
event.last_seen = time.end_time
else:
# if these values were not set use now
event.created_at = datetime.utcnow()
event.modified_on = datetime.utcnow()
event.first_seen = datetime.utcnow()
event.last_seen = datetime.utcnow()
# TODO find a way to add this in stix / read it out
event.status = 'Confirmed'
event.analysis = 'Unknown'
event.risk = 'Undefined'
event.properties.is_shareable = True
event.properties.is_rest_instert = True
if not event.first_seen:
event.first_seen = event.created_at
if not event.last_seen:
event.last_seen = event.modified_on
return event
def create_event(self, stix_package, user, ignore_id=False):
# First process the header
event = self.__map_stix_package_header(stix_package, user)
set_properties(event)
# TODO Make relations
set_extended_logging(event, user, user.group)
seen_conditions = dict()
# First handle observables of the package
if stix_package.observables:
for observable in stix_package.observables.observables:
obs = self.cybox_mapper.create_observable(
observable, event, user, event.tlp_level_id, False,
seen_conditions)
if obs:
event.observables.append(obs)
# Then handle indicators
if stix_package.indicators:
# process iocs
for indicator in stix_package.indicators:
child = self.create_indicator(indicator, event, user)
if child:
event.indicators.append(child)
else:
raise Exception('None Value')
# Process incidents
if stix_package.incidents:
for incident in stix_package.incidents:
raise StixCelsusMapperException(
u'Incidents are not yet supported')
if not ignore_id:
self.event_controller.insert_event(user, event, False, False)
return event
def get_tlp_id_from_markings(self, indicator):
handling = indicator.handling
if handling:
markings = handling.markings
for marking in markings:
for stucture in marking.marking_structures:
if isinstance(stucture, TLPMarkingStructure):
color = stucture.color
tlp_id = TLP.get_by_value(color.title())
return tlp_id
return None
def create_indicator(self, indicator, event, user):
ce1sus_indicator = Indicator()
set_properties(ce1sus_indicator)
if indicator.id_:
ce1sus_indicator.uuid = extract_uuid(indicator.id_)
else:
ce1sus_indicator.uuid = uuid4()
ce1sus_indicator.title = indicator.title
ce1sus_indicator.description = indicator.description
ce1sus_indicator.originating_group = self.__convert_info_soucre(
indicator.producer)
# TODO: Check if condifdence is in the supported range like Low Hiogh medium or even numbers?!
ce1sus_indicator.confidence = indicator.confidence
sightings = self.__convert_sightings(indicator.sightings, user)
if sightings:
ce1sus_indicator.sightings = sightings
# TODO: Add indicator types
# TODO: markings
if indicator.kill_chain_phases:
# TODO: Kill Chains
pass
# Note observable is actually observables[0]
if indicator.observables:
tlp_id = self.get_tlp_id_from_markings(indicator)
if tlp_id is None:
tlp_id = event.tlp_level_id
for observable in indicator.observables:
observable = self.cybox_mapper.create_observable(
observable, event, user, tlp_id, True)
if observable:
observable.event = None
observable.event_id = None
ce1sus_indicator.observables.append(observable)
set_extended_logging(ce1sus_indicator, user, user.group)
return ce1sus_indicator
def __init__(self, config, session=None):
BaseController.__init__(self, config, session)
self.cybox_mapper = CyboxMapper(config, session)
self.indicator_controller = IndicatorController(config, session)
self.event_controller = EventController(config, session)
self.fh = FileHandler()
class Ce1susStixMapper(BaseController):
def __init__(self, config, session=None):
BaseController.__init__(self, config, session)
self.cybox_mapper = CyboxMapper(config, session)
self.indicator_controller = IndicatorController(config, session)
self.event_controller = EventController(config, session)
self.fh = FileHandler()
def init(self):
self.seen_groups = list()
self.seen_observables = list()
def __map_stix_header(self, event):
self.init()
stix_header = STIXHeader(title=event.title)
stix_header.description = event.description
stix_header.short_description = event.title
identifiy = self.create_stix_identity(event)
time = self.cybox_mapper.get_time(produced_time=event.created_at,
received_time=event.modified_on)
info_source = InformationSource(identity=identifiy, time=time)
stix_header.information_source = info_source
# Add TLP
marking = Marking()
marking_spec = MarkingSpecification()
marking.add_marking(marking_spec)
tlp_marking_struct = TLPMarkingStructure()
tlp_marking_struct.color = event.tlp.upper()
tlp_marking_struct.marking_model_ref = 'http://govcert.lu/en/docs/POL_202_V2.2_rfc2350.pdf'
marking_spec.marking_structures = list()
marking_spec.marking_structures.append(tlp_marking_struct)
stix_header.handling = marking
return stix_header
def create_stix_identity(self, obj):
idenfitier = 'ce1sus:Group-{0}'.format(obj.creator_group.uuid)
if idenfitier in self.seen_groups:
identity = Identity()
identity.idref = idenfitier
else:
identity = Identity()
identity.id_ = idenfitier
identity.name = obj.creator_group.name
self.seen_groups.append(idenfitier)
return identity
def add_custom_property(self, instance, attribute):
if not instance.custom_properties:
cust_props = CustomProperties()
instance.custom_properties = cust_props
custom_prop = Property()
custom_prop.name = attribute.definition.name
custom_prop.description = attribute.definition.description
custom_prop._value = attribute.value
if hasattr(custom_prop._value, 'condition'):
custom_prop._value.condition = self.get_condition(attribute)
if hasattr(custom_prop, 'condition'):
custom_prop.condition = self.get_condition(attribute)
instance.custom_properties.append(custom_prop)
def create_EmailAddress(self, attribute):
email = EmailAddress(attribute.value)
email.condition = self.get_condition(attribute)
return email
def __check_set_email_header(self, cybox_email):
if not cybox_email.header:
cybox_email.header = EmailHeader()
def populate_email(self, cybox_email, attribute):
# returns a cybox email object out of a ce1sus object
def_name = attribute.definition.name
if def_name == 'email_attachment_file_name':
attachment = File()
attachment.file_name = attribute.value
attachment.file_name.condition = self.get_condition(attribute)
# cybox_email.attachments = Attachments()
# cybox_email.attachments.append(File)
elif def_name == 'email_bcc':
self.__check_set_email_header(cybox_email)
if not cybox_email.header.bcc:
cybox_email.header.bcc = EmailRecipients()
cybox_email.header.bcc.append(self.create_EmailAddress(attribute))
elif def_name == 'email_cc':
self.__check_set_email_header(cybox_email)
if not cybox_email.header.cc:
cybox_email.header.cc = EmailRecipients()
cybox_email.header.bcc.append(self.create_EmailAddress(attribute))
elif def_name == 'email_errors_to':
self.__check_set_email_header(cybox_email)
self.set_check_attr(cybox_email, 'header.errors_to', attribute)
elif def_name == 'email_message_id':
self.__check_set_email_header(cybox_email)
self.set_check_attr(cybox_email, 'header.message_id', attribute)
elif def_name == 'email_mime_version':
self.__check_set_email_header(cybox_email)
self.set_check_attr(cybox_email, 'header.mime_version', attribute)
elif def_name == 'email_raw_body':
self.set_check_attr(cybox_email, 'raw_body', attribute)
elif def_name == 'email_raw_header':
self.set_check_attr(cybox_email, 'raw_header', attribute)
elif def_name == 'email_reply_to':
if not cybox_email.header.in_reply_to:
self.__check_set_email_header(cybox_email)
cybox_email.header.in_reply_to = EmailRecipients()
cybox_email.header.in_reply_to.append(
self.create_EmailAddress(attribute))
elif def_name == 'email_server':
self.set_check_attr(cybox_email, 'email_server', attribute)
elif def_name == 'email_subject':
self.set_check_attr(cybox_email, 'subject', attribute)
elif def_name == 'email_from':
self.__check_set_email_header(cybox_email)
if not cybox_email.header.from_:
cybox_email.header.from_ = self.create_EmailAddress(attribute)
elif def_name == 'email_to':
self.__check_set_email_header(cybox_email)
if not cybox_email.header.to:
cybox_email.header.to = EmailRecipients()
cybox_email.header.to.append(self.create_EmailAddress(attribute))
elif def_name == 'email_x_mailer':
self.set_check_attr(cybox_email, 'header.x_mailer', attribute)
elif def_name == 'email_x_originating_ip':
self.set_check_attr(cybox_email, 'header.x_originating_ip',
attribute)
elif 'hash' in def_name:
raise CyboxMapperException('Not defined')
elif def_name == 'email_link':
if not cybox_email.links:
cybox_email.links = Links()
cybox_email.links.append(Link(attribute.value))
elif def_name == 'email_send_date':
cybox_email.date = attribute.value
elif def_name == 'email_in_reply_to':
self.__check_set_email_header(cybox_email)
cybox_email.header.in_reply_to = attribute.value
else:
raise CyboxMapperException('Not defined for {0}'.format(def_name))
def set_check_attr(self, cybox_obj, proerty_name, attribute):
# TODO make this correct
cybox_value = None
if '.' in proerty_name:
properies = proerty_name.split('.')
prop1 = getattr(cybox_obj, properies[0])
if prop1:
cybox_value = getattr(prop1, properies[1])
else:
pass
else:
cybox_value = getattr(cybox_obj, proerty_name, None)
if cybox_value:
cybox_value.condition = self.get_condition(attribute)
else:
setattr(cybox_obj, proerty_name, attribute.value)
cybox_value = getattr(cybox_obj, proerty_name, None)
if hasattr(cybox_value, 'condition'):
cybox_value.condition = self.get_condition(attribute)
def map_attribtue(self, instance, attribute):
definition_name = attribute.definition.name
if hasattr(instance, definition_name.lower()):
setattr(instance, definition_name.lower(), attribute.value)
value = getattr(instance, definition_name.lower())
value.condition = self.get_condition(attribute)
else:
if 'hash' in definition_name:
if attribute.condition.value:
if attribute.condition.value:
if attribute.condition.value == "Equals":
exact = True
else:
exact = False
else:
exact = False
else:
exact = True
h = Hash(attribute.value, exact=exact)
h.condition = attribute.condition
if instance.hashes is None:
instance.hashes = HashList()
instance.hashes.append(h)
pass
elif 'email' in definition_name:
self.populate_email(instance, attribute)
elif definition_name == 'url':
instance.type_ = URI.TYPE_URL
instance.value = attribute.value
instance.value.condition = self.get_condition(attribute)
elif 'Full_Path' and isinstance(instance, Process):
# TODO: check why this is set?!?
pass
elif definition_name == 'WindowsRegistryKey_Key':
instance.key = attribute.value
instance.key.condition = self.get_condition(attribute)
elif definition_name == 'WindowsRegistryKey_Hive':
instance.hive = attribute.value
instance.hive.condition = self.get_condition(attribute)
elif 'WindowsRegistryKey_RegistryValue' in definition_name:
value = RegistryValue()
if definition_name == 'WindowsRegistryKey_RegistryValue_Data':
value.data = attribute.value
value.data.condition = self.get_condition(attribute)
elif definition_name == 'WindowsRegistryKey_RegistryValue_Data':
value.name = attribute.value
value.name.condition = self.get_condition(attribute)
if not instance.values:
instance.values = RegistryValues()
instance.values.append(value)
instance.data = attribute.value
elif definition_name == 'ipv4_addr':
instance.category = definition_name.replace('_', '-')
instance.address_value = attribute.value
instance.address_value.condition = self.get_condition(
attribute)
elif definition_name == 'DomainName_Value':
instance.value = attribute.value
elif definition_name == 'Raw_Artifact':
path = '{0}/{1}'.format(self.fh.get_base_path(),
attribute.value)
if isfile(path):
with open(path, 'r') as f:
bin_string = f.read()
instance.data = bin_string
# TODO find the corect type
instance.type_ = Artifact.TYPE_GENERIC
instance.packaging.append(Base64Encoding())
else:
instance.data = 'MIA'
instance.type_ = Artifact.TYPE_GENERIC
instance.packaging.append(Base64Encoding())
elif definition_name == 'content_type':
instance.type_ = attribute.value
elif definition_name == 'URIType':
instance.type_ = attribute.value
elif not attribute.definition.cybox_std:
self.add_custom_property(instance, attribute)
elif isinstance(instance,
NetworkConnection) and definition_name in [
'is_type', 'Port'
]:
# TODO: check why this is set?!?
pass
else:
raise Ce1susStixMapperException(
'Cannot map {1} on object type {0}'.format(
instance.__class__.__name__, definition_name))
def get_condition(self, attribute):
if attribute.condition:
return attribute.condition.value
else:
# default condition is equals
return 'Equals'
def create_object(self, ce1sus_object, event_permissions, user):
definition_name = ce1sus_object.definition.name
obj = Object()
identifier = 'ce1sus:Object-{0}'.format(ce1sus_object.uuid)
obj.id_ = identifier
# try to find automatic the object container
try:
clazz = get_class(
'cybox.objects.{0}_object'.format(definition_name.lower()),
definition_name)
instance = clazz()
if definition_name == 'Disk':
# TODO: check why this must be set stix bug?
setattr(instance, 'type_', None)
except ImportError:
if definition_name == 'WinEventLog':
instance = WinEventLog()
# TODO: check why this must be set stix bug?
setattr(instance, 'type_', None)
elif definition_name == 'UserAccount':
instance = UserAccount()
elif definition_name == 'WinService':
instance = WinService()
elif definition_name == 'WindowsRegistryKey':
instance = WinRegistryKey()
elif definition_name == 'NetworkConnection':
instance = NetworkConnection()
elif definition_name == 'WinVolume':
instance = WinVolume()
# TODO: check why this must be set stix bug?
setattr(instance, 'drive_type', None)
elif definition_name == 'WinKernelHook':
instance = WinKernelHook()
elif definition_name == 'WinDriver':
instance = WinDriver()
elif definition_name == 'DomainName':
instance = DomainName()
# TODO: try to map it manually
elif definition_name == 'email':
instance = EmailMessage()
else:
raise Ce1susStixMapperException(
'Required to map manually {0}'.format(definition_name))
obj.properties = instance
attributes = ce1sus_object.get_attributes_for_permissions(
event_permissions, user)
for attribute in attributes:
self.map_attribtue(instance, attribute)
rel_objects = ce1sus_object.get_related_objects_for_permissions(
event_permissions, user)
for rel_object in rel_objects:
ob = self.create_object(rel_object.object, event_permissions, user)
if ob:
rel_obj = self.create_related_object(rel_object,
event_permissions, user)
# cybox_rel_object = RelatedObject(properties=ob.properties, relationship=rel_object.relation)
obj.related_objects.append(rel_obj)
return obj
def create_related_object(self, rel_object, event_permissions, user):
ob = self.create_object(rel_object.object, event_permissions, user)
rel_obj = RelatedObject(properties=ob.properties,
relationship=rel_object.relation)
rel_obj.id_ = ob.id_
for relobj in rel_object.object.related_objects:
rel_child_obj = self.create_related_object(relobj,
event_permissions, user)
rel_obj.related_objects.append(rel_child_obj)
return rel_obj
def create_composed_observable(self, ce1sus_composition, event_permissions,
user):
composition = ObservableComposition()
composition.operator = ce1sus_composition.operator
ce1sus_obs = ce1sus_composition.get_observables_for_permissions(
event_permissions, user)
for ce1sus_ob in ce1sus_obs:
obs = self.create_observable(ce1sus_ob, event_permissions, user)
composition.observables.append(obs)
return composition
def create_observable(self, ce1sus_obs, event_permissions, user):
identifier = 'ce1sus:Observable-{0}'.format(ce1sus_obs.uuid)
cybox_observable = Observable()
if identifier in self.seen_observables:
# if I've seen the uuid then make a reference insead
cybox_observable.idref = identifier
else:
self.seen_observables.append(identifier)
cybox_observable.id_ = identifier
cybox_observable.title = ce1sus_obs.title
cybox_observable.description = ce1sus_obs.description
if ce1sus_obs.object:
cybox_obj = self.create_object(ce1sus_obs.object,
event_permissions, user)
cybox_observable.object_ = cybox_obj
elif ce1sus_obs.observable_composition:
cybox_obj_composition = self.create_composed_observable(
ce1sus_obs.observable_composition, event_permissions, user)
cybox_observable.observable_composition = cybox_obj_composition
return cybox_observable
def create_indicator(self, ce1sus_indicator, event_permissions, user):
indicator = Indicator()
indicator.id_ = 'ce1sus:Indicator-{0}'.format(ce1sus_indicator.uuid)
indicator.title = ce1sus_indicator.title
indicator.description = ce1sus_indicator.description
indicator.short_description = ce1sus_indicator.short_description
if ce1sus_indicator.confidence:
indicator.confidence = ce1sus_indicator.confidence.title()
else:
indicator.confidence = 'Low'
# TODO: handling
# TODO: markings
for type_ in ce1sus_indicator.types:
indicator.add_indicator_type(type_.name)
if ce1sus_indicator.operator:
indicator.observable_composition_operator = ce1sus_indicator.operator
# Todo Add confidence
# indicator_attachment.confidence = "Low"
creator = self.create_stix_identity(ce1sus_indicator)
time = self.cybox_mapper.get_time(
produced_time=ce1sus_indicator.created_at)
info_source = InformationSource(identity=creator, time=time)
indicator.producer = info_source
observables = ce1sus_indicator.get_observables_for_permissions(
event_permissions, user)
for obs in observables:
cybox_obs = self.create_observable(obs, event_permissions, user)
indicator.add_observable(cybox_obs)
valid_time = ValidTime(start_time=ce1sus_indicator.created_at,
end_time=ce1sus_indicator.created_at)
indicator.add_valid_time_position(valid_time)
return indicator
def create_stix(self, event, user):
stix_package = STIXPackage()
stix_package.id_ = 'ce1sus:Event-{0}'.format(event.uuid)
stix_header = self.__map_stix_header(event)
stix_package.stix_header = stix_header
event_permissions = self.event_controller.get_event_user_permissions(
event, user)
# observables
if event.observables:
for observable in event.get_observables_for_permissions(
event_permissions, user):
cybox_obs = self.create_observable(observable,
event_permissions, user)
stix_package.add_observable(cybox_obs)
# indicators
if event.indicators:
indicators = event.get_indicators_for_permissions(
event_permissions, user)
else:
# generate indicators
indicators = self.indicator_controller.get_generic_indicators(
event, user)
for indicator in indicators:
stix_indicator = self.create_indicator(indicator,
event_permissions, user)
stix_package.add_indicator(stix_indicator)
return stix_package
class Maintenance(object):
"""
Maintenance class
"""
def __init__(self, config):
self.config = config
self.session_manager = SessionManager(config)
directconnection = self.session_manager.connector.get_direct_session()
self.relation_controller = RelationController(config, directconnection)
self.attribute_definition_controller = AttributeDefinitionController(
config, directconnection)
self.object_definition_controller = ObjectDefinitionController(
config, directconnection)
self.event_controller = EventController(config, directconnection)
self.user_controller = UserController(config, directconnection)
self.reference_controller = ReferencesController(
config, directconnection)
self.conditions_controller = ConditionController(
config, directconnection)
self.verbose = False
# set maintenance user
user_uuid = config.get('ce1sus', 'maintenaceuseruuid', None)
if None:
raise MaintenanceException(
'maintenaceuseruuid was not defined in config')
try:
self.user = self.user_controller.get_user_by_uuid(user_uuid)
except ControllerNothingFoundException:
raise MaintenanceException(
'Cannot find maintenance user with uuid {0}'.format(user_uuid))
except ControllerException as error:
raise MaintenanceException(error)
def drop_relations(self, event_uuid=''):
try:
if event_uuid:
if self.verbose:
print 'Dropping relations for event {0}'.format(event_uuid)
try:
event = self.event_controller.get_event_by_uuid(event_uuid)
self.relation_controller.remove_relations_for_event(event)
except ControllerNothingFoundException:
raise MaintenanceException(
'Event with uuid "{0}" cannot be found'.format(
event_uuid))
else:
if self.verbose:
print 'Dropping all relations'
self.relation_controller.clear_relations_table()
except ControllerException as error:
raise MaintenanceException(error)
def rebuild_relations(self, event_uuid='', from_date=''):
try:
if event_uuid:
if self.verbose:
print '(Re)Creation relations for event {0}'.format(
event_uuid)
try:
event = self.event_controller.get_event_by_uuid(event_uuid)
# drop relations for event
self.relation_controller.remove_relations_for_event(event)
flat_attributes = self.relation_controller.get_flat_attributes_for_event(
event)
self.relation_controller.generate_bulk_attributes_relations(
event, flat_attributes, True)
except ControllerNothingFoundException:
raise MaintenanceException(
'Event with uuid "{0}" cannot be found'.format(
event_uuid))
else:
# drop all relations
self.relation_controller.clear_relations_table()
if from_date:
from_date = stringToDateTime(from_date)
if self.verbose:
print '(Re)Creation all relations for the events created from {0} on'.format(
from_date)
events = self.event_controller.get_all_from(from_date)
else:
if self.verbose:
print '(Re)Creation all relations'
events = self.event_controller.get_all()
for event in events:
if self.verbose:
print 'Rebuild relations for event {0}'.format(
event.identifier)
flat_attributes = self.relation_controller.get_flat_attributes_for_event(
event)
self.relation_controller.generate_bulk_attributes_relations(
event, flat_attributes, False)
self.relation_controller.relation_broker.do_commit(True)
except ControllerException as error:
raise MaintenanceException(error)
def fix_chksums(self):
try:
if self.verbose:
print 'Checking object definitions'
objects = self.object_definition_controller.get_all_object_definitions(
)
not_matching = list()
for obj in objects:
if obj.chksum != gen_obj_chksum(obj):
not_matching.append(obj)
if self.verbose:
print 'Found {0} not object matching object chksums'.format(
len(not_matching))
# fix checksums
for not_matching_obj in not_matching:
not_matching_obj.chksum = gen_obj_chksum(not_matching_obj)
self.object_definition_controller.obj_def_broker.update(
not_matching, False)
if self.verbose:
print 'Checking attribute definitions'
# find attribute chksums not matching the one stored
not_matching = list()
objects = self.attribute_definition_controller.get_all_attribute_definitions(
)
for obj in objects:
if obj.chksum != gen_attr_chksum(obj):
not_matching.append(obj)
if self.verbose:
print 'Found {0} attributes not matching object chksums'.format(
len(not_matching))
for not_matching_attr in not_matching:
not_matching_attr.chksum = gen_attr_chksum(not_matching_attr)
self.attribute_definition_controller.attr_def_broker.update(
not_matching_attr, False)
self.attribute_definition_controller.attr_def_broker.do_commit(
True)
except (ControllerException, BrokerException) as error:
raise MaintenanceException(error)
def register_handler(self, modulename, type_='attributes', classname=None):
if not classname:
classname = modulename.title().replace('handler', 'Handler')
if self.verbose:
print 'Registering handler {0}'.format(classname)
modulename = u'{0}'.format(modulename)
clazz = get_class(u'ce1sus.handlers.{0}.{1}'.format(type_, modulename),
classname)
instance = clazz()
if isinstance(instance, HandlerBase):
if self.verbose:
print u'Adding handler {0}.{1}'.format(modulename, classname)
uuid = instance.get_uuid()
description = instance.get_description()
if type_ == 'attributes':
self.attribute_definition_controller.register_handler(
uuid, u'{0}.{1}'.format(modulename, classname),
description)
elif type_ == 'references':
self.reference_controller.register_handler(
uuid, u'{0}.{1}'.format(modulename, classname),
description)
else:
raise MaintenanceException(
'Type {0} for handlers is unknown'.format(type_))
return uuid
# TODO check if the required files are available
else:
raise MaintenanceException(
'Class {0}.{1} does not implement HandlerBase'.format(
modulename, classname))
def dump_definitions(self, dump_def, dump_dest):
# check if file exists
if isfile(dump_dest):
raise MaintenanceException(
'File {0} is already existing'.format(dump_dest))
dump = list()
if dump_def == 'attributes':
attributes = self.attribute_definition_controller.get_all_attribute_definitions(
)
for attribute in attributes:
dump.append(attribute.to_dict(True, True))
elif dump_def == 'objects':
obejcts = self.object_definition_controller.get_all_object_definitions(
)
for obj in obejcts:
dump.append(obj.to_dict(True, True))
elif dump_def == 'references':
ref_defs = self.reference_controller.get_reference_definitions_all(
)
for ref_def in ref_defs:
dump.append(ref_def.to_dict(True, True))
elif dump_def == 'conditions':
conditions = self.conditions_controller.get_all_conditions()
for condition in conditions:
dump.append(condition.to_dict(True, True))
elif dump_def == 'types':
types = self.attribute_definition_controller.get_all_types()
for type_ in types:
dump.append(type_.to_dict(True, True))
else:
raise MaintenanceException(
'No definition assigned to {0}. It can either be attributes or objects'
.format(dump_def))
# open an dump to file
dump_file = open(dump_dest, 'w+')
dump_file.write(json.dumps(dump))
dump_file.close()
class Scheduler(object):
def __init__(self, config):
self.config = config
self.session_manager = SessionManager(config)
directconnection = self.session_manager.connector.get_direct_session()
self.process_controller = ProcessController(config, directconnection)
self.server_controller = SyncServerController(config, directconnection)
self.misp_converter = MispConverter(config, None, None, None,
directconnection)
dump = config.get('MISPAdapter', 'dump', False)
file_loc = config.get('MISPAdapter', 'file', None)
self.misp_converter.dump = dump
self.misp_converter.file_location = file_loc
self.misp_ctrl = MISPAdapter(config, directconnection)
user_uuid = config.get('ce1sus', 'maintenaceuseruuid', None)
self.user_controller = UserController(config, directconnection)
self.event_controller = EventController(config, directconnection)
self.group_broker = self.user_controller.broker_factory(GroupBroker)
self.mail_controller = MailController(config, directconnection)
self.ce1sus_adapter = Ce1susAdapter(config, directconnection)
if None:
raise SchedulerException(
'maintenaceuseruuid was not defined in config')
try:
self.user = self.user_controller.get_user_by_uuid(user_uuid)
except ControllerNothingFoundException:
raise SchedulerException(
'Cannot find maintenance user with uuid {0}'.format(user_uuid))
except ControllerException as error:
raise SchedulerException(error)
def __publish_event(self, item, event):
# server publishing
if item.server_details:
server_details = item.server_details
else:
raise SchedulerException('Server could not be defined')
if server_details.type == 'MISP':
self.__push_misp(item, event)
elif server_details.type == 'Ce1sus':
self.__push_ce1sus(item, event)
else:
raise SchedulerException('Server type {0} is unkown'.format(
server_details.type))
def __push_ce1sus(self, item, event, dologin=True):
try:
self.ce1sus_adapter.server_details = item.server_details
if dologin:
self.ce1sus_adapter.login()
rem_event = self.ce1sus_adapter.get_event_by_uuid(
event.uuid, False, False)
# check if server has the event already
if rem_event:
# update the event
self.ce1sus_adapter.update_event(event, True, True)
else:
# insert the event
self.ce1sus_adapter.insert_event(event, True, True)
except Ce1susAdapterException as error:
raise SchedulerException(error)
try:
if dologin:
self.ce1sus_adapter.logout()
except Ce1susAdapterException as error:
raise SchedulerException(error)
def __push_misp(self, item, event):
# set the parameters for misp
self.misp_converter.api_key = item.server_details.user.api_key
self.misp_converter.api_url = item.server_details.baseurl
self.misp_converter.tag = item.server_details.name
# cehck if event can be seen else ignore, normally should not happen
if not self.misp_ctrl.is_event_viewable(event,
item.server_details.user):
# Ignore push
return True
# use the other function as it filters unwanted stuff out
event_xml = self.misp_ctrl.make_misp_xml(event,
item.server_details.user)
try:
self.misp_converter.push_event(event_xml)
self.process_controller.process_finished_success(item, self.user)
except MispConverterException as error:
# TODO Log
self.process_controller.process_finished_in_error(item, self.user)
raise SchedulerException(
'Error during push of event {0} on server with url {1} with error {2}'
.format(event.uuid, item.server_details.baseurl, error))
def __publish(self, event_uuid, item_array):
event = self.event_controller.get_event_by_uuid(event_uuid)
item_publish = False
for item in item_array:
try:
if item.server_details:
# do the sync only for this server
self.__publish_event(item, event)
else:
# it is to send emails
self.__send_mails(event, item.type_)
# set event as published only if it is the furst publication
if item.type_ in [
ProcessType.PUBLISH, ProcessType.PUBLISH_UPDATE
]:
item_publish = True
# remove item from queue
self.process_controller.process_finished_success(
item, self.user)
except (ControllerException, BrokerException) as error:
self.process_controller.process_finished_in_error(
item, self.user)
# only update the publish date once
if item_publish:
try:
event.last_publish_date = datetime.utcnow()
self.event_controller.update_event(self.user, event, True,
True)
except (ControllerException, BrokerException) as error:
raise SchedulerException(error)
def __send_mails(self, event, type_):
# send mails only if plugin is enabled
if self.mail_controller.mail_handler:
try:
seen_mails = list()
# send mails for the users who wants them
users = self.user_controller.get_all_notifiable_users()
for user in users:
# prevent double emails
if user.email in seen_mails:
continue
else:
# only send mails to users who can see the event!!!
if is_event_viewable_user(event, user):
if user.gpg_key:
seen_mails.append(user.email)
self.__send_mail(type_, event, user, None,
True)
else:
# only send email when white
if event.tlp_level_id >= 3:
seen_mails.append(user.email)
self.__send_mail(type_, event, user, None,
False)
# send the mails for the groups which want them
groups = self.group_broker.get_all_notifiable_groups()
for group in groups:
if group.email in seen_mails:
continue
else:
if is_event_viewable_group(event, group):
# only send mails to users who can see the event!!!
seen_mails.append(group.email)
if group.gpg_key:
self.__send_mail(type_, event, None, group,
True)
else:
# only send email when white
if event.tlp_level_id >= 3:
self.__send_mail(type_, event, None, group,
False)
except (ControllerException, BrokerException) as error:
raise SchedulerException(error)
def __get_mail(self, event, type_, user, group):
if type_ in [ProcessType.PUBLISH, ProcessType.REPUBLISH]:
return self.mail_controller.get_publication_mail(
event, user, group)
elif type_ in [
ProcessType.PUBLISH_UPDATE, ProcessType.REPUBLISH_UPDATE
]:
return self.mail_controller.get_publication_update_mail(
event, user, group)
elif type_ == ProcessType.PROPOSAL:
return self.mail_controller.get_proposal_mail(event, user, group)
else:
raise SchedulerException(u'{0} is undefined'.format(type_))
def __send_mail(self, type_, event, user, group, encrypt):
# determine if it is an update
mail = self.__get_mail(event, type_, user, group)
mail.encrypt = encrypt
# send mail
self.mail_controller.send_mail(mail)
def __pull(self, item):
if item.server_details:
# do the sync only for this server
if item.server_details.type == 'MISP':
# set the parameters for misp
self.misp_converter.api_key = item.server_details.user.api_key
self.misp_converter.api_url = item.server_details.baseurl
self.misp_converter.tag = item.server_details.name
# fetch one event from misp
misp_event_xml = self.misp_converter.get_xml_event(
item.event_uuid)
# make insert/merge
try:
self.misp_ctrl.ins_merg_event(item.server_details,
misp_event_xml,
item.server_details.user,
self.user)
self.process_controller.process_finished_success(
item, self.user)
except MISPAdapterException as error:
raise SchedulerException(error)
# TODO dump xml or log it in browser
elif item.server_details.type.lower() == 'ce1sus':
self.ce1sus_adapter.server_details = item.server_details
self.ce1sus_adapter.login()
rem_json = self.ce1sus_adapter.get_event_by_uuid(
item.event_uuid, True, True, True, True)
# check if event exists
try:
event = self.event_controller.get_event_by_uuid(
item.event_uuid)
# merge the event fetched
owner = is_event_owner(event, item.server_details.user)
event = self.ce1sus_adapter.assembler.update_event(
event, rem_json, item.server_details.user, owner, True)
self.__insert_provenance_report(event, item.server_details)
self.event_controller.update_event(self.user, event, True,
True)
except ControllerNothingFoundException:
event = self.ce1sus_adapter.assembler.assemble_event(
rem_json, item.server_details.user, True, True, False)
event.properties.is_validated = False
self.__insert_provenance_report(event, item.server_details)
self.event_controller.insert_event(self.user, event, True,
True)
except ControllerException as error:
self.ce1sus_adapter.logout()
raise SchedulerException(error)
self.ce1sus_adapter.logout()
self.process_controller.process_finished_success(
item, self.user)
pass
else:
raise SchedulerException('Server type {0} is unkown'.format(
item.server_details.type))
else:
# do the sync for all servers which are pull servers
pass
def __insert_provenance_report(self, event, server_details):
report = Report()
report.event = event
self.ce1sus_adapter.set_extended_logging(report, server_details.user,
server_details.user.group,
True)
reference = Reference()
definition = self.misp_converter.reference_definitions_broker.get_by_uuid(
'dee2aa50-874e-4a92-9fd0-441171e76585')
reference.definition = definition
reference.value = '{0}/#/events/event/{1}'.format(
server_details.baseurl, event.uuid)
report.references.append(reference)
event.reports.append(report)
self.ce1sus_adapter.set_extended_logging(reference,
server_details.user,
server_details.user.group,
True)
def __push(self, item):
if item.server_details:
# do the sync only for this server
if item.server_details.type == 'MISP':
event = self.event_controller.get_event_by_uuid(
item.event_uuid)
self.__push_misp(item, event)
elif item.server_details.type == 'Ce1sus':
self.ce1sus_adapter.server_details = item.server_details
self.ce1sus_adapter.login()
event = self.event_controller.get_event_by_uuid(
item.event_uuid)
try:
try:
rem_event = self.ce1sus_adapter.get_event_by_uuid(
item.event_uuid, False, False, False, False)
self.__insert_provenance_report(
event, item.server_details)
owner = is_event_owner(event, item.server_details.user)
event = self.ce1sus_adapter.assembler.update_event(
event, rem_event, item.server_details.user, owner,
True)
self.__insert_provenance_report(
event, item.server_details)
self.ce1sus_adapter.update_event(event, True, True)
except Ce1susAdapterNothingFoundException:
event.properties.validated = False
self.__insert_provenance_report(
event, item.server_details)
self.ce1sus_adapter.insert_event(event, True, True)
self.process_controller.process_finished_success(
item, item.server_details.user)
except (BrokerException, ControllerException,
Ce1susAdapterException) as error:
raise SchedulerException(error)
finally:
self.ce1sus_adapter.logout()
else:
raise SchedulerException('Server type {0} is unkown'.format(
item.server_details.type))
else:
# do the sync for all servers which are push servers
pass
def process(self):
# get all items
items = self.process_controller.get_scheduled_process_items(self.user)
# sort out publications
publications = list()
remaining = list()
for item in items:
self.process_controller.process_task(item, self.user)
if item.type_ in [
ProcessType.PUBLISH, ProcessType.PUBLISH_UPDATE,
ProcessType.REPUBLISH, ProcessType.REPUBLISH_UPDATE
]:
publications.append(item)
else:
remaining.append(item)
for item in remaining:
try:
if item.type_ == ProcessType.PULL:
self.__pull(item)
elif item.type_ == ProcessType.PUSH:
self.__push(item)
elif item.type_ == ProcessType.RELATIONS:
# this is only possible if it is a ce1sus internal server, hence server details = none
if item.server_details:
raise SchedulerException(
'For relations the server details have to be none')
else:
# TODO: make relations
pass
elif item.type_ == ProcessType.PROPOSAL:
self.__proposal(item)
except SchedulerException:
self.process_controller.process_finished_in_error(
item, self.user)
grouped_publications = dict()
for item in publications:
items = grouped_publications.get(item.event_uuid, None)
if items is None:
grouped_publications[item.event_uuid] = list()
grouped_publications[item.event_uuid].append(item)
for key, item_array in grouped_publications.iteritems():
try:
self.__publish(key, item_array)
except SchedulerException:
self.process_controller.process_finished_in_error(
item, self.user)