def _make_stix_package_for_attached_file(self, file_, feed): # package ID作成 package_id = self.generator.create_id(prefix='Package') # 添付ファイルの中身を読み込み base64 で encode with open(file_.file_path, 'rb') as fp: content = base64.b64encode(fp.read()) # content作成 marking_specification_content = self._make_marking_specification_statement( MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX, content.decode('utf-8')) # filename作成 marking_specification_file_name = self._make_marking_specification_statement( MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX, file_.file_name) # header 作成 stix_header = STIXHeader() stix_header.handling = self._get_stix_header_marking(feed) stix_header.handling.add_marking(marking_specification_content) stix_header.handling.add_marking(marking_specification_file_name) stix_header.title = file_.file_name stix_header.description = 'File "%s" encoded in BASE64.' % ( file_.file_name) # Information Source 格納 stix_header.information_source = self._make_information_source() # package作成 stix_package = STIXPackage(id_=package_id) stix_package.timestamp = datetime.datetime.now( tz=pytz.timezone(feed.user.timezone)) stix_package.stix_header = stix_header return stix_package
def _make_stix_package(self, feed, indicators=[], ttps=[], tas=[]): user_timezone = pytz.timezone(feed.user.timezone) # package ID作成 package_id = self.generator.create_id(prefix='Package') # package作成 stix_package = STIXPackage(id_=package_id) stix_package.timestamp = datetime.datetime.now(tz=user_timezone) # header格納 stix_package.stix_header = self._get_stix_header(feed) # indicators 格納 # web 画面から取得した indicators (json) から stix indicators 作成する stix_indicators = Indicators() for indicator_json in indicators: indicator = CommonExtractor.get_indicator_from_json( indicator_json, user_timezone) if indicator is not None: stix_indicators.append(indicator) stix_package.indicators = stix_indicators # ExploitTargets格納 stix_exploit_targets = ExploitTargets() for ttp_json in ttps: et = CommonExtractor.get_exploit_target_from_json(ttp_json) if et is not None: stix_exploit_targets.append(et) stix_package.exploit_targets = stix_exploit_targets # ThreatActors 格納 for ta_json in tas: value = ta_json['value'] if SNSConfig.get_cs_custid( ) is not None and SNSConfig.get_cs_custkey() is not None: ta = self.get_ta_from_crowd_strike(value) if ta is None: # ATT&CK から ThreatActor 取得する ta = self.get_ta_from_attck(value) else: ta = self.get_ta_from_attck(value) stix_package.add_threat_actor(ta) # 添付ファイル用の STIX 作成する for file_ in feed.files.all(): attach_file_stix_package = self._make_stix_package_for_attached_file( file_, feed) self.attachment_files.append(attach_file_stix_package) # 添付ファイル用の STIX を Related Pacakge に追加する stix_package.add_related_package(attach_file_stix_package.id_) return stix_package
def download_stix(request): request.session.set_expiry(SESSION_EXPIRY) error_ = check_allow_l1_view(request) if error_ is not None: return error_ try: # Ctirsクラスのインスタンスを作成 ctirs = Ctirs(request) # package_id取得 package_id = get_l1_package_id(request) # apiからcontent取得 dict_ = ctirs.get_stix_file_stix(package_id) stix_package = STIXPackage.from_dict(dict_) # XML変換した文字列をStringIO化する(その際にUNICODEに変換) output = io.StringIO() output.write(stix_package.to_xml()) filename = '%s.xml' % (package_id) # response作成 response = HttpResponse(output.getvalue(), content_type='application/xml') response['Content-Disposition'] = 'attachment; filename=%s' % ( filename) return response except Exception: # エラーページ return error_page(request)
def update_stix_file_v1(pacakge_id, content): # 文字列イメージの引数のxmlをxml形式にして、インデントを揃えてダウンロードする file_path = get_file_name_from_package_id(pacakge_id) + '.xml' output = io.StringIO(content) # xml整形して再書き込み p = STIXPackage.from_xml(output) # 中身とファイル名を返却 return p.to_xml(), file_path
def _get_stix_content_dict(stix_file): if stix_file.version.startswith('2.') == True: #STIX 2.x return json.loads(stix_file.content.read()) else: #STIX 1.x stix_package = STIXPackage.from_xml(stix_file.origin_path) return stix_package.to_dict()
def get_attach_file(api_user, stix_id): attachment_stix_dir = Feed.get_attach_stix_dir_path(stix_id) if os.path.exists(attachment_stix_dir): # ここのファイル名とファイルパスを返却する try: # 1dir 1file なので最初の要素をファイル名とする attach_file = AttachFile() attach_file.file_name = Feed.get_attach_file_name(stix_id) attach_file.package_id = stix_id attach_file.save() return attach_file except IndexError: # dir はあるが fileが存在しない場合に発生する # 処理は続行する pass else: # dir が存在しないので作成 os.mkdir(attachment_stix_dir) attachement_cached_stix_file_path = rs.get_stix_file_path(api_user, stix_id) try: # Attachement STIX Package parse attachement_stix_package = STIXPackage.from_xml(attachement_cached_stix_file_path) # marking から file_name, content 取得 file_name = None content = None try: markings = attachement_stix_package.stix_header.handling.marking except AttributeError: return None for marking in markings: marking_structure = marking.marking_structures[0] if isinstance(marking_structure, SimpleMarkingStructure): statement = marking_structure.statement if statement.startswith(const.MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX): file_name = statement[len(const.MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX + ': '):] elif statement.startswith(const.MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX): content_str = statement[len(const.MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX + ': '):] # content は base64 で decode する content = base64.b64decode(content_str) if (file_name is None) or (content is None): return None except Exception as e: import traceback traceback.print_exc() raise e # ファイル保存 file_path = attachment_stix_dir + os.sep + file_name file_path = file_path.encode('utf-8') with open(file_path, 'wb') as fp: fp.write(content) attach_file = AttachFile() attach_file.file_name = file_name attach_file.package_id = stix_id attach_file.save() return attach_file
def upload_xml(self, *vpath, **params): user = None try: user = self.user_controller.get_user_by_username( self.get_user().username) except ControllerException as error: self.logger.error(error) raise HTTPError(400, error.message) input_json = self.get_json() filename = input_json['name'] self.logger.info( 'Starting to import xml form file {0}'.format(filename)) data = input_json['data']['data'] complete = params.get('complete', False) inflated = params.get('inflated', False) xml_string = base64.b64decode(data) # STIX wants lxml instead of xml xml = etree.fromstring(xml_string) stix_package = STIXPackage.from_xml(xml) try: event = self.stix_mapper.map_stix_package(stix_package, user) event.properties.is_validated = False self.event_controller.insert_event(user, event, True, True) event_permissions = self.event_controller.get_event_user_permissions( event, user) cherrypy.response.headers[ 'Content-Type'] = 'application/json; charset=UTF-8' return json.dumps( event.to_dict(complete, inflated, event_permissions, user)) except ControllerIntegrityException as error: self.logger.debug(error) event = self.stix_mapper.map_stix_package(stix_package, user, False, True) local_event = self.event_controller.get_event_by_uuid(event.uuid) event_permissions = self.event_controller.get_event_user_permissions( event, user) merged_event = self.merger.merge_event(local_event, event, user, event_permissions) self.event_controller.update_event(user, merged_event, True, True) cherrypy.response.headers[ 'Content-Type'] = 'application/json; charset=UTF-8' return json.dumps( merged_event.to_dict(complete, inflated, event_permissions, user)) except ControllerNothingFoundException as error: self.logger.error(error) raise HTTPError(404, '{0}'.format(error.message)) except ControllerException as error: self.logger.error(error) raise HTTPError(400, '{0}'.format(error.message))
def update_stix_file_v1(pacakge_id, content): # 文字列イメージの引数のxmlをxml形式にして、インデントを揃えてダウンロードする file_path = get_file_name_from_package_id(pacakge_id) + '.xml' output = io.StringIO(content) # xml整形して再書き込み p = STIXPackage.from_xml(output) xml = p.to_xml() if isinstance(xml, bytes): xml = xml.decode() # 中身とファイル名を返却 return xml, file_path
def upload_misp(self, package_id): stix_file = StixFiles.objects.get(package_id=package_id) stix_package = STIXPackage.from_xml(stix_file.content) misp_event = load_stix(stix_package) tag = self.get_tlp_tag(stix_package) if tag is not None: misp_event.add_tag(tag) resp = self.py_misp.add_event(misp_event) if resp.has_key('Event') == True: return resp else: raise Exception(str(resp['errors']))
def __init__(self, feed=None, stix_file_path=None, indicators=None, ttps=None, tas=None): super(FeedStix, self).__init__() self.stix_package = None self.attachment_files = [] if feed is not None: self.stix_package = self._make_stix_package( feed, indicators, ttps, tas) elif stix_file_path is not None: self.stix_package = STIXPackage.from_xml(stix_file_path)
def _get_event(self, uuid, version): payload = {} payload['uuid'] = uuid payload['returnFormat'] = version header = self.header.copy() if version == 'stix': header['Accept'] = 'application/xml' else: header['Accept'] = 'application/json' resp_text = self._post_misp(header, payload) if version == 'stix': with io.StringIO(resp_text) as fp: package = STIXPackage.from_xml(fp) return package else: return json.loads(resp_text)
def upload_misp(self, package_id): stix_file = StixFiles.objects.get(package_id=package_id) if stix_file.version.startswith('1.'): content = stix_file.content else: content = io.StringIO(stix_file.get_slide_12()) stix_package = STIXPackage.from_xml(content) misp_event = load_stix(stix_package) tag = self.get_tlp_tag(stix_package) if tag is not None: misp_event.add_tag(tag) resp = self.py_misp.add_event(misp_event) if ('Event' in resp): return resp else: raise Exception(str(resp['errors']))
def _make_stix_package(self, origin_feed, post, creator=None): # package ID作成 package_id = self.generator.create_id(prefix='Package') # package作成 stix_package = STIXPackage(id_=package_id) stix_package.timestamp = datetime.datetime.now(tz=pytz.timezone(origin_feed.user.timezone)) # header格納 stix_package.stix_header = self._get_stix_header(origin_feed, post, creator) # Comment元の Feed の Package ID を Related Package に追加する stix_package.add_related_package(origin_feed.package_id) return stix_package
def get_raw_stix(request): request.session.set_expiry(SESSION_EXPIRY) # GET以外はエラー if request.method != 'GET': r = {'status': 'NG', 'message': 'Invalid HTTP method'} return JsonResponse(r, safe=False) r = check_allow_sharing_view(request) if r is not None: return r try: # package_id取得 package_id = get_sharing_ajax_get_raw_stix_package_id(request) # Ctirsクラスのインスタンスを作成 ctirs = Ctirs(request) # STIXファイルの中身を取得 j = ctirs.get_stix_file_stix(package_id) # STIX 2.x であるかの判定を行う v2_flag = _is_stix2_(j) if v2_flag: # 返却json r = { 'status': 'OK', 'message': 'Success.', 'stix_version': '2.0', 'contents': j } else: stix_package = STIXPackage.from_dict(j) # 返却json xml = stix_package.to_xml() if isinstance(xml, bytes): xml = xml.decode() r = { 'status': 'OK', 'message': 'Success.', 'stix_version': '1.2', 'contents': xml } except Exception as e: traceback.print_exc() r = {'status': 'NG', 'message': str(e)} finally: return JsonResponse(r, safe=False)
def get_draw_data(request): request.session.set_expiry(SESSION_EXPIRY) # GET以外はエラー if request.method != 'GET': r = {'status': 'NG', 'message': 'Invalid HTTP method'} return JsonResponse(r, safe=False) r = check_allow_sharing_view(request) if r is not None: return r try: # package_id名取得 package_id = get_sharing_ajax_get_draw_data_package_id(request) # community名取得 community = get_sharing_ajax_get_draw_data_community(request) # Ctirsクラスのインスタンスを作成 ctirs = Ctirs(request) # GetPolicy相当呼び出し rules = get_policy(community) # REST_API から STIX の json イメージを取得 dict_ = ctirs.get_stix_file_stix(package_id) # STIX 2.x であるかの判定を行う v2_flag = _is_stix2_(dict_) r = {'status': 'OK', 'rules': rules, 'message': 'Success.'} if v2_flag: # STIX 2.x の場合 r['json'] = dict_ r['stix_version'] = '2.0' else: # STIX 1.x の場合 # json から XML イメージを返却 xml = STIXPackage.from_dict(dict_).to_xml() if isinstance(xml, bytes): xml = xml.decode() r['xml'] = xml r['stix_version'] = '1.2' except Exception as e: traceback.print_exc() r = {'status': 'NG', 'message': str(e)} finally: return JsonResponse(r, safe=False)
def get_package_info(request): request.session.set_expiry(SESSION_EXPIRY) # GET以外はエラー if request.method != 'GET': r = {'status': 'NG', 'message': 'Invalid HTTP method'} return JsonResponse(r, safe=False) # activeユーザー以外はエラー if not request.user.is_active: r = {'status': 'NG', 'message': 'You account is inactive.'} return JsonResponse(r, safe=False) try: # package_id取得 package_id = get_package_id(request) # l1情報取得 l1_type_list = get_package_l1_info(request, package_id) # description 取得 try: # Ctirsクラスのインスタンスを作成 ctirs = Ctirs(request) # STIXイメージ取得 dict_ = ctirs.get_stix_file_stix(package_id) stix_package = STIXPackage.from_dict(dict_) description = stix_package.stix_header.description.value except BaseException: # エラー時は空白 description = '' # 返却データ r = {'status': 'OK', 'description': description} # l1情報追加 for l1_type in l1_type_list: type_, values = l1_type r[type_] = values except Exception as e: print('Excepton:' + str(e)) r = {'status': 'NG', 'message': str(e)} finally: return JsonResponse(r, safe=False)
def capsulize_patch(self, pkg_id, enable_bfs=False): contents = [] pkg = STIXPackage( id_=pkg_id, stix_header=generate_stix_header(self) ) def pkg_dispatch(eo): if isinstance(eo.obj, stixbase.DBStixBase): PACKAGE_ADD_DISPATCH[eo.ty](pkg, eo.obj._object) else: PACKAGE_ADD_DISPATCH[eo.ty](pkg, eo.obj) if enable_bfs: queue = [self.id_] completed_ids = set() while queue: eo_id = queue.pop() if eo_id in completed_ids: continue completed_ids.add(eo_id) if self.id_ == eo_id: eo = self #must do this as self may be a version other than latest else: try: eo = EdgeObject.load(eo_id, self.filters) except EdgeError: continue pkg_dispatch(eo) contents.append(eo) queue.extend([edge.id_ for edge in eo.edges]) else: pkg_dispatch(self) contents.append(self) return pkg, contents
def get_package_bean_v1(stix_file_path): doc = STIXPackage.from_xml(stix_file_path) try: package_bean = StixFiles.PackageBean() package_bean.is_post_sns = True package_bean.is_created_by_sns = False sns_type = None if _is_produced_by_stip_sns_v1(doc): package_bean.is_created_by_sns = True sns_type = _get_stip_sns_type_v1(doc) if sns_type != StixFiles.STIP_SNS_TYPE_ORIGIN: package_bean.is_post_sns = False try: package_bean.related_packages = [] for related_package in doc.related_packages: package_bean.related_packages.append(related_package.item.id_) except TypeError: package_bean.related_packages = None package_bean.package_id = doc.id_ package_bean.version = doc._version package_bean.produced = _get_produced_time_stix_1_x(doc) package_bean.package_name = doc.stix_header.title package_bean.sns_type = sns_type try: package_bean.description = doc.stix_header.description.value if package_bean.description is None: package_bean.description = '' except BaseException: package_bean.description = '' _set_stix_bean_from_doc_v1(package_bean, doc) if package_bean.sns_user_name == '': package_bean.sns_user_name = _get_sns_user_name_from_instance( package_bean.sns_instance) return package_bean except Exception: pass
def get_attach_file(api_user, stix_id, version): attachment_stix_dir = Feed.get_attach_stix_dir_path(stix_id) if os.path.exists(attachment_stix_dir): try: attach_file = AttachFile() attach_file.file_name = Feed.get_attach_file_name(stix_id) attach_file.package_id = stix_id attach_file.save() return attach_file except IndexError: pass else: os.mkdir(attachment_stix_dir) attachement_cached_stix_file_path = rs.get_stix_file_path( api_user, stix_id) try: if version.startswith('v1'): attachement_stix_package = STIXPackage.from_xml( attachement_cached_stix_file_path) file_name = None content = None try: markings = attachement_stix_package.stix_header.handling.marking except AttributeError: return None for marking in markings: marking_structure = marking.marking_structures[0] if isinstance(marking_structure, SimpleMarkingStructure): statement = marking_structure.statement if statement.startswith( const. MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX ): file_name = statement[len( const. MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX + ': '):] elif statement.startswith( const. MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX ): content_str = statement[len( const. MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX + ': '):] content = base64.b64decode(content_str) if (file_name is None) or (content is None): return None elif version.startswith('v2'): with open(attachement_cached_stix_file_path, 'r', encoding='utf-8') as fp: attachment_bundle = json.load(fp) file_name = None content = None for o_ in attachment_bundle['objects']: if o_['type'] != const.STIP_STIX2_X_STIP_SNS_TYPE: continue if o_[const. STIP_STIX2_PROP_TYPE] != const.STIP_STIX2_SNS_POST_TYPE_ATTACHMENT: continue stip_sns_attachment = o_[const.STIP_STIX2_PROP_ATTACHMENT] content_str = stip_sns_attachment[ const.STIP_STIX2_SNS_ATTACHMENT_CONTENT_KEY] content = base64.b64decode(content_str) file_name = stip_sns_attachment[ const.STIP_STIX2_SNS_ATTACHMENT_FILENAME_KEY] break if (file_name is None) or (content is None): return None except Exception as e: import traceback traceback.print_exc() raise e file_path = attachment_stix_dir + os.sep + file_name file_path = file_path.encode('utf-8') with open(file_path, 'wb') as fp: fp.write(content) attach_file = AttachFile() attach_file.file_name = file_name attach_file.package_id = stix_id attach_file.save() return attach_file
def _get_stix_header_marking(self, feed, creator=None): if creator is None: user = feed.user else: user = creator s = STIXPackage() if user.region is not None: country_name_code = user.region.country_code admin_area_name_code = user.region.code else: country_name_code = user.country_code admin_area_name_code = None industry_type = ais.OTHER organisation_name = ais.OTHER ais_tlp = feed.tlp # REDの場合はAISではエラーとなるのでNoneとする if ais_tlp == 'RED': ais_tlp = None ais.add_ais_marking(s, False, 'EVERYONE', ais_tlp, country_name_code=country_name_code, country_name_code_type='ISO_3166-1_alpha-2', admin_area_name_code=admin_area_name_code, organisation_name=organisation_name, industry_type=industry_type, admin_area_name_code_type="ISO_3166-2", ) ais_marking_specification = s.stix_header.handling.marking[0] # Marking作成 marking = Marking() marking.add_marking(ais_marking_specification) # 汎用のTLPmarking_specification作成 marking_specification = self._get_tlp_marking_structure(feed) marking.add_marking(marking_specification) # Critical Infrastructure marking_critical_infrastructure = self._make_marking_specification_statement(const.STIP_SNS_CI_KEY, user.ci) marking.add_marking(marking_critical_infrastructure) # スクリーン名 marking_screen_name = self._make_marking_specification_statement(const.STIP_SNS_SCREEN_NAME_KEY, user.get_screen_name()) marking.add_marking(marking_screen_name) # username marking_user_name = self._make_marking_specification_statement(const.STIP_SNS_USER_NAME_KEY, user.username) marking.add_marking(marking_user_name) # 会社名 marking_affiliation = self._make_marking_specification_statement(const.STIP_SNS_AFFILIATION_KEY, user.affiliation) marking.add_marking(marking_affiliation) # region code if user.region is not None: marking_region_code = self._make_marking_specification_statement(const.STIP_SNS_REGION_CODE_KEY, user.region.code) marking.add_marking(marking_region_code) # Sharing Range if feed.sharing_range_type == const.SHARING_RANGE_TYPE_KEY_ALL: sharing_range = 'CIC Community' elif feed.sharing_range_type == const.SHARING_RANGE_TYPE_KEY_GROUP: sharing_range = 'Group: %s' % (feed.sharing_group.en_name) elif feed.sharing_range_type == const.SHARING_RANGE_TYPE_KEY_PEOPLE: sharing_range = 'People: ' feed.save() for sharing_stip_user in feed.tmp_sharing_people: sharing_range += sharing_stip_user.username sharing_range += ',' # 最後の改行を取り除く sharing_range = sharing_range[:-1] marking_sharing_range = self._make_marking_specification_statement('Sharing Range', sharing_range) marking.add_marking(marking_sharing_range) # referred_url if feed.referred_url is not None: marking_referred_url = self._make_marking_specification_statement(const.STIP_SNS_REFERRED_URL_KEY, feed.referred_url) marking.add_marking(marking_referred_url) # stix2_package_id if feed.stix2_package_id is not None and len(feed.stix2_package_id) != 0: marking_stix2_package_id = self._make_marking_specification_statement(const.STIP_SNS_STIX2_PACKAGE_ID_KEY, feed.stix2_package_id) marking.add_marking(marking_stix2_package_id) return marking
def transform(data, new_only=True): """ transform - The transforms are source specific. Source: http://www.malwaredomainlist.com/hostslist/mdl.xml data - must be source xml converted to a dictionary :param data: :param new_only: :return: """ # Input validation if not isinstance(data, dict): return False work = [] history = db('local_file', 'history', ADPTR_SRC_ID) value2key = db('local_file', 'value_to_key', 'values') items = data.get('rss', {}).get('channel', {}).get('item') if items: for item in items: guid = item.get('guid', {}).get('#text') if guid: # Check to see if this item has been process before # if not, add to work if guid in history: if not new_only: work.append(item) else: work.append(item) db('local_file', 'history', ADPTR_SRC_ID, {guid: { 'date': str(datetime.now()) }}) if work: ### Generate STIXPackage and STIXHeader set_ns_stix(ADPTR_NS_STIX) set_ns_cybox(ADPTR_NS_CYBOX) STIXPackage._version = ADPTR_VER_STIX pkg = STIXPackage() src_info, value2key = gen_info_src({}, 'www.malwaredomainlist.com', value2key) hdr = STIXHeader() hdr.title = data.get('rss', {}).get('channel', {}).get('title') hdr.description = data.get('rss', {}).get('channel', {}).get('description') hdr.information_source = src_info pkg.stix_header = hdr for item in work: key = item.get('guid', {}).get('#text') # Decompose data description tmp = [x.strip() for x in item.get('description').split(',')] decomp = {} for x in tmp: k, v = x.split(':') decomp.update({k.strip(): v.strip()}) # Generate STIX Indicator ind, history = gen_indicator(item, key, history) ind.producer = src_info ind.short_description = 'MDL RefID: %s | %s' % ( key, decomp.get('Description')) # Decompose host host = decomp.get('Host') uri = None file_ = None if '/' in host: host, uri = host.split('/', 1) # TODO: parse out file Name if host: # Generate Cybox HostName obj = Hostname() obj.is_domain_name = True obj.naming_system = 'DNS' obj.hostname_value = host ob, value2key = gen_CyboxOb(obj, host, value2key) ob.title = 'HostName: %s' % obj.hostname_value ind.add_observable(CyboxOb(idref=ob.id_)) pkg.add_observable(ob) if uri: # Generate Cybox URI obj = URI() obj.type_ = URI.TYPE_URL url = AnyURI('%s/%s' % (host, uri)) obj.value = url ob, value2key = gen_CyboxOb(obj, url, value2key) ob.title = 'URL: %s' % url ind.add_observable(CyboxOb(idref=ob.id_)) pkg.add_observable(ob) if file_: obj = File() ip = decomp.get('IP address') if ip: obj_ip = Address() if isIPv4(ip): obj_ip.category = Address.CAT_IPV4 elif isIPv6(ip): obj_ip.category = Address.CAT_IPV6 else: break obj_ip.is_source = True obj_ip.address_value = ip # if obj_host: # obj_ip.add_related(obj_host, # ObjectRelationship.TERM_RESOLVED_TO, # inline=False) ob = CyboxOb(obj_ip) ob.title = 'IP: %s' % ip ind.add_observable(CyboxOb(idref=ob.id_)) pkg.add_observable(ob) asn = decomp.get('ASN') if asn: obj_asn = Address() obj_asn.category = Address.CAT_ASN obj_asn.address_value = asn # if obj_host: # obj_asn.add_related(obj_host, # ObjectRelationship.TERM_CONNECTED_TO, # inline=False) # if obj_ip: # obj_asn.add_related(obj_ip, # ObjectRelationship.TERM_CONNECTED_TO, # inline=False) ob = CyboxOb(obj_asn) ob.title = 'ASN: %s' % ip ind.add_observable(CyboxOb(idref=ob.id_)) pkg.add_observable(ob) pkg.add_indicator(ind) db('local_file', 'value_to_key', 'values', value2key) db('local_file', 'history', ADPTR_SRC_ID, history) return pkg
def set_alchemy_nodes(aj, content): if content['version'].startswith('2.'): is_stix_v2 = True else: is_stix_v2 = False package_name = content['package_name'] if is_stix_v2: if isinstance(content, dict): package = content['dict'] else: package = json.loads(content['dict']) else: package = STIXPackage.from_dict(content['dict']) if is_stix_v2: stix_header = None else: stix_header = package.stix_header if is_stix_v2: an_package_id = package['id'] else: an_package_id = convert_valid_node_id(package.id_) an_header_id = an_package_id if is_stix_v2: indicators = [] observables = [] campaigns = [] threat_actors = [] coas = [] identies = [] malwares = [] sightings = [] intrusion_sets = [] attack_patterns = [] relationships = [] reports = [] tools = [] vulnerabilities = [] custom_objects = [] locations = [] opinions = [] notes = [] language_contents = [] ''' x_stip_snses = [] ''' ttps = None ets = None incidents = None for o_ in package['objects']: object_type = o_['type'] if object_type == 'indicator': indicators.append(o_) elif object_type == 'identity': identies.append(o_) elif object_type == 'observed-data': observables.append(o_) elif object_type == 'malware': malwares.append(o_) elif object_type == 'sighting': sightings.append(o_) elif object_type == 'intrusion-set': intrusion_sets.append(o_) elif object_type == 'threat-actor': threat_actors.append(o_) elif object_type == 'attack-pattern': attack_patterns.append(o_) elif object_type == 'campaign': campaigns.append(o_) elif object_type == 'relationship': relationships.append(o_) elif object_type == 'course-of-action': coas.append(o_) elif object_type == 'report': reports.append(o_) elif object_type == 'tool': tools.append(o_) elif object_type == 'vulnerability': vulnerabilities.append(o_) elif object_type == 'location': locations.append(o_) elif object_type == 'opinion': opinions.append(o_) elif object_type == 'note': notes.append(o_) elif object_type == 'language-content': language_contents.append(o_) ''' elif object_type == 'x-stip-sns': x_stip_snses.append(o_) ''' if object_type == 'x-stip-sns': continue elif object_type.startswith('x-'): custom_objects.append(o_) else: indicators = package.indicators observables = package.observables campaigns = package.campaigns ttps = package.ttps threat_actors = package.threat_actors ets = package.exploit_targets coas = package.courses_of_action incidents = package.incidents identies = None malwares = None sightings = None intrusion_sets = None attack_patterns = None relationships = None reports = None tools = None vulnerabilities = None locations = None opinions = None notes = None language_contents = None custom_objects = None if not is_stix_v2: an = AlchemyNode(an_header_id, 'Header', package_name, stix_header.description, cluster=an_package_id) aj.add_json_node(an) if indicators is not None: if not is_stix_v2: an_indicators_id = an_package_id + '--indicators' else: an_indicators_id = None indicators_values = [] for indicator in indicators: l = set_alchemy_node_indicator(aj, indicator, an_indicators_id, is_stix_v2, an_package_id) indicators_values.extend(l) if not is_stix_v2: an = AlchemyNode(an_indicators_id, 'Indicators', 'Indicators', '', cluster=an_package_id) an.set_value(get_observable_value_string_from_list(indicators_values)) aj.add_json_node(an) ae = AlchemyEdge(an_header_id, an_indicators_id, LABEL_EDGE) aj.add_json_edge(ae) if observables is not None: if not is_stix_v2: an_observables_id = an_package_id + '--observables' else: an_observables_id = None obsevables_values = [] for observed_data in observables: l = set_alchemy_node_observable(aj, observed_data, an_observables_id, is_stix_v2, an_package_id) obsevables_values.extend(l) if not is_stix_v2: an = AlchemyNode(an_observables_id, 'Observables', 'Observables', '', cluster=an_package_id) an.set_value(get_observable_value_string_from_list(obsevables_values)) aj.add_json_node(an) ae = AlchemyEdge(an_header_id, an_observables_id, LABEL_EDGE) aj.add_json_edge(ae) if campaigns is not None: if not is_stix_v2: an_campaigns_id = an_package_id + '--campaigns' an = AlchemyNode(an_campaigns_id, 'Campaigns', 'Campaigns', '', cluster=an_package_id) aj.add_json_node(an) ae = AlchemyEdge(an_header_id, an_campaigns_id, LABEL_EDGE) aj.add_json_edge(ae) else: an_campaigns_id = None for campaign in campaigns: set_alchemy_node_campaign(aj, campaign, an_campaigns_id, is_stix_v2, an_package_id) if threat_actors is not None: if not is_stix_v2: an_tas_id = an_package_id + '--tas' an = AlchemyNode(an_tas_id, 'Threat_Actors', 'Threat_Actors', '', cluster=an_package_id) aj.add_json_node(an) ae = AlchemyEdge(an_header_id, an_tas_id, LABEL_EDGE) aj.add_json_edge(ae) else: an_tas_id = None for threat_actor in threat_actors: set_alchemy_node_threat_actor(aj, threat_actor, an_tas_id, is_stix_v2, an_package_id) if coas is not None: if not is_stix_v2: an_coas_id = an_package_id + '--coas' an = AlchemyNode(an_coas_id, 'Courses_Of_Action', 'Courses_Of_Action', '', cluster=an_package_id) aj.add_json_node(an) ae = AlchemyEdge(an_header_id, an_coas_id, LABEL_EDGE) aj.add_json_edge(ae) else: an_coas_id = None for coa in coas: set_alchemy_node_coa(aj, coa, an_coas_id, is_stix_v2, an_package_id) if ttps is not None: an_ttps_id = an_package_id + '--ttps' an = AlchemyNode(an_ttps_id, 'TTPs', 'TTPs', '', cluster=an_package_id) aj.add_json_node(an) ae = AlchemyEdge(an_header_id, an_ttps_id, LABEL_EDGE) aj.add_json_edge(ae) for ttp in ttps: set_alchemy_node_ttp(aj, ttp, an_ttps_id, an_package_id) if ets is not None: an_ets_id = an_package_id + '--ets' an = AlchemyNode(an_ets_id, 'Exploit_Targets', 'Exploit_Targets', '', cluster=an_package_id) aj.add_json_node(an) ae = AlchemyEdge(an_header_id, an_ets_id, LABEL_EDGE) aj.add_json_edge(ae) for et in ets: set_alchemy_node_et(aj, et, an_ets_id, an_package_id) if incidents is not None: an_incidents_id = an_package_id + '--incidents' an = AlchemyNode(an_incidents_id, 'Incidents', 'Incidents', '', cluster=an_package_id) aj.add_json_node(an) ae = AlchemyEdge(an_header_id, an_incidents_id, LABEL_EDGE) aj.add_json_edge(ae) for incident in incidents: set_alchemy_node_incident(aj, incident, an_incidents_id, an_package_id) if identies is not None: for identity in identies: set_alchemy_node_identity(aj, identity, an_package_id) if malwares is not None: for malware in malwares: set_alchemy_node_malware(aj, malware, an_package_id) if sightings is not None: for sighting in sightings: set_alchemy_node_sighting(aj, sighting, an_package_id) if intrusion_sets is not None: for intrusion_set in intrusion_sets: set_alchemy_node_intrusion_set(aj, intrusion_set, an_package_id) if attack_patterns is not None: for attack_pattern in attack_patterns: set_alchemy_node_attack_pattern(aj, attack_pattern, an_package_id) if reports is not None: for report in reports: set_alchemy_node_report(aj, report, an_package_id) if tools is not None: for tool in tools: set_alchemy_node_tool(aj, tool, an_package_id) if vulnerabilities is not None: for vulnerability in vulnerabilities: set_alchemy_node_vulnerability(aj, vulnerability, an_package_id) if locations is not None: for location in locations: set_alchemy_node_location(aj, location, an_package_id) if opinions is not None: for opinion in opinions: set_alchemy_node_opinion(aj, opinion, an_package_id) if notes is not None: for note in notes: set_alchemy_node_note(aj, note, an_package_id) if custom_objects is not None: for custom_object in custom_objects: set_alchemy_node_custom_object(aj, custom_object, an_package_id) ''' if x_stip_snses is not None: for x_stip_sns in x_stip_snses: set_alchemy_node_x_stip_sns(aj, x_stip_sns, an_package_id) ''' if relationships is not None: for relationship in relationships: set_alchemy_node_relationship(aj, relationship, an_package_id) return
def call_jira(request): try: # JIRA が import されていない場合は何もしない if imported_jira is None: rsp = {} return JsonResponse(rsp) # feed情報取得 feed_file_name_id = request.GET['feed_id'] package_id_arg = request.GET['package_id'] feed = Feed.get_feeds_from_package_id(request.user, package_id_arg) # JIRA instance proxies = System.get_request_proxies() j = JIRA(server=SNSConfig.get_jira_host(), proxies=proxies, basic_auth=(SNSConfig.get_jira_username(), SNSConfig.get_jira_password())) # issues作成 issue = j.create_issue(project=SNSConfig.get_jira_project(), summary=feed.title, description=feed.post, issuetype={'name': SNSConfig.get_jira_type()}) # 添付があればそれもつける for attach_file in feed.files.all(): file_path = Feed.get_attach_file_path(attach_file.package_id) j.add_attachment(issue=issue, attachment=file_path, filename=str(attach_file.file_name)) # STIX添付 stix_package = STIXPackage.from_xml(feed.stix_file_path) package_id = stix_package.id_ stix_file_name = '%s.xml' % (package_id) j.add_attachment(issue=issue, attachment=feed.stix_file_path, filename=stix_file_name) # CSV添付 # CSVの中身を取得する content = get_csv_content(feed_file_name_id) csv_attachment = io.StringIO() csv_attachment.write(content) csv_file_name = '%s.csv' % (package_id) j.add_attachment(issue=issue, attachment=csv_attachment, filename=csv_file_name) # PDF添付 feed_pdf = FeedPDF(feed, stix_package) pdf_attachment = io.BytesIO() feed_pdf.make_pdf_content(pdf_attachment, feed) pdf_file_name = '%s.pdf' % (package_id) j.add_attachment(issue=issue, attachment=pdf_attachment, filename=pdf_file_name) # isssue番号返却 url = SNSConfig.get_jira_host( ) + '/projects/' + SNSConfig.get_jira_project() + '/issues/' + str( issue) rsp = { 'issues': str(issue), 'url': url, } return JsonResponse(rsp) except Exception as e: traceback.print_exc() return HttpResponseServerError(str(e))
def get_package_bean(stix_file_path): package_bean = StixFiles.PackageBean() #STIX 1.1 parse try: doc = STIXPackage.from_xml(stix_file_path) package_bean.is_post_sns = True package_bean.is_created_by_sns = False sns_type = None #S-TIP SNS で作成された STIX であるか? if is_produced_by_stip_sns(doc) == True: #SNS 産である package_bean.is_created_by_sns = True sns_type = get_stip_sns_type(doc) #origin 投稿以外は表示しない if sns_type != StixFiles.STIP_SNS_TYPE_ORIGIN: package_bean.is_post_sns = False #realted_packages探す try: package_bean.related_packages = [] for related_package in doc.related_packages: package_bean.related_packages.append(related_package.item.id_) except TypeError: package_bean.related_packages = None package_bean.package_id = doc.id_ package_bean.version = doc._version package_bean.produced = get_produced_time_stix_1_x(doc) package_bean.package_name = doc.stix_header.title package_bean.sns_type = sns_type try: package_bean.description = doc.stix_header.description.value if package_bean.description is None: package_bean.description = '' except: package_bean.description = '' #S-TIP SNS 作成の STIX から pacakge_bean の値をセットする set_stix_bean_from_doc(package_bean, doc) #SNS 産以外は sns_user_name が設定されていないので instance 名から取得する if package_bean.sns_user_name == '': package_bean.sns_user_name = get_sns_user_name_from_instance( package_bean.sns_instance) return package_bean except Exception: pass #STIX 2.0 parse try: with codecs.open(stix_file_path, 'r', 'utf-8') as fp: content = fp.read() doc = json.loads(content) package_bean.package_name = None #最初に見つかったtypeがreportのnameをpackage_nameとする #また、STIX 2.0 では package の timestampの格納場所がないのでNoneとする produced_str = None for d in doc['objects']: if d['type'] == 'report': package_bean.package_name = d['name'] produced_str = d['created'] package_bean.package_id = doc['id'] if doc.has_key('spec_version') == True: package_bean.version = doc['spec_version'] else: #STIX 2.1 には spec_version がない package_bean.version = '2.1' #Produced Time は Report の produced time if produced_str is not None: package_bean.produced = stix2_str_to_datetime(produced_str) else: package_bean.produced = datetime.datetime.now() package_bean.is_post_sns = True package_bean.is_created_by_sns = False package_bean.related_packages = None return package_bean except Exception as e: traceback.print_exc() raise Exception('Can\'t parse STIX. ' + e.message)
def create_feeds_record(api_user, package_id, uploader_id, produced_str): # RS から取得した STIX から stix_package 取得する stix_file_path = rs.get_stix_file_path(api_user, package_id) stix_package = STIXPackage.from_xml(stix_file_path, encoding='utf-8') # Feed情報を STIX,RS の API から取得する feed = Feed() feed.package_id = package_id feed.filename_pk = rs.convert_package_id_to_filename(package_id) # STIX から表示情報を取得する bean = Feed.get_stip_from_stix_package(stix_package) if bean.is_sns: # SNS 産 STIX である if bean.instance == SNSConfig.get_sns_identity_name(): # 現在稼働中インスタンスと同一 try: # STIX 定義の username が存在する feed.user = STIPUser.objects.get(username=bean.user_name) # 表示はローカル DB から取得する Feed.set_screen_value_from_local_db(feed, bean) except BaseException: # STIX 定義の username が存在しない → N/A アカウント feed.user = Feed.get_na_account() # 表示はSTIX File から取得する Feed.set_screen_value_from_stix(feed, bean) # すでにユーザーが削除されている feed.is_valid_user = False else: # 現在稼働中インスタンスと異なる try: # インスタンス名と同じアカウント feed.user = STIPUser.objects.get(username=bean.instance) # 表示はSTIX File から取得する Feed.set_screen_value_from_stix(feed, bean) except BaseException: # インスタンス名と同じアカウントが存在しない → N/A アカウント feed.user = Feed.get_na_account() # 表示はSTIX File から取得する Feed.set_screen_value_from_stix(feed, bean) else: # SNS 産 STIX ではない if bean.instance is not None: # instance がある instance_user_name = bean.instance.replace(' ', '') try: # インスタンス名と同じアカウント feed.user = STIPUser.objects.get(username=instance_user_name) # 表示はローカル DB から取得する Feed.set_screen_value_from_local_db(feed, bean) except BaseException: # インスタンス名と同じアカウントが存在しない → N/A アカウント feed.user = Feed.get_na_account() # 表示は bean.instance [bean.instance] feed.screen_name = bean.instance feed.screen_instance = bean.instance else: # instance がない # N/A アカウント feed.user = Feed.get_na_account() # 表示はローカル DB(N/Aアカウント) から取得する Feed.set_screen_value_from_local_db(feed, bean) feed.date = Feed.get_datetime_from_string(produced_str) feed.post = stix_package.stix_header.description if feed.post is None: feed.post = '' # Attachement Files 情報取得 if Feed.is_stip_sns_stix_package(stix_package): # S-TIP SNS 作成 STIX if stix_package.related_packages is not None: # 一度 feed をsave()する feed.save() # Related_packages は SNS STIX 以外の可能性もある for related_package in stix_package.related_packages: # attachement は attachdirにいれるべきその時のファイル名は attachment_stix_idであるべき attach_file = Feed.get_attach_file(api_user, related_package.item.id_) # attach_file が None の場合は Attach File ではない if attach_file is None: continue feed.files.add(attach_file) feed.save() feed.title = stix_package.stix_header.title feed.stix_file_path = stix_file_path try: uploader_stipuser = STIPUser.objects.get(id=uploader_id) feed.tlp = Feed.get_ais_tlp_from_stix_header(stix_package.stix_header, uploader_stipuser.tlp) if feed.tlp is None: # 取得ができなかった場合は default TLP の AMBER feed.tlp = 'AMBER' except BaseException: # uploader_profile が存在しない場合は default TLP の AMBER feed.tlp = 'AMBER' sharing_range_info = Feed.get_sharing_range_from_stix_header(stix_package.stix_header) if isinstance(sharing_range_info, list): # sharing_range_info の中は STIPUser list feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_PEOPLE feed.save() for stip_user in sharing_range_info: feed.sharing_people.add(stip_user) elif isinstance(sharing_range_info, Group): feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_GROUP feed.sharing_group = sharing_range_info else: feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_ALL # feed.package_id = package_id if bean.region_code is not None: feed.region_code = bean.region_code else: if feed.user.region is not None: feed.region_code = feed.user.region.code else: feed.region_code = '' if bean.ci is not None: feed.ci = bean.ci else: feed.ci = feed.user.ci if bean.referred_url is not None: feed.referred_url = bean.referred_url if bean.stix2_package_id is not None: feed.stix2_package_id = bean.stix2_package_id feed.save() return feed
def create_feeds_record_v1(api_user, package_id, uploader_id, produced_str, version): stix_file_path = rs.get_stix_file_path(api_user, package_id) stix_package = STIXPackage.from_xml(stix_file_path) feed = Feed() feed.stix_version = version feed.package_id = package_id feed.filename_pk = rs.convert_package_id_to_filename(package_id) bean = Feed.get_stip_from_stix_package_v1(stix_package) if bean.is_sns: if bean.instance == SNSConfig.get_sns_identity_name(): try: feed.user = STIPUser.objects.get(username=bean.user_name) Feed.set_screen_value_from_local_db(feed, bean) except BaseException: feed.user = Feed.get_na_account() Feed.set_screen_value_from_stix(feed, bean) feed.is_valid_user = False else: try: feed.user = STIPUser.objects.get(username=bean.instance) Feed.set_screen_value_from_stix(feed, bean) except BaseException: feed.user = Feed.get_na_account() Feed.set_screen_value_from_stix(feed, bean) else: def _get_match_stip_user(username): try: username = username.replace(' ', '') return STIPUser.objects.get(username=username) except STIPUser.DoesNotExist: return None except Exception as e: raise e def _get_match_stip_user_from_bean(bean): ret_instance = None ret_tool = None if bean.instance: ret_instance = bean.instance stip_user = _get_match_stip_user(bean.instance) if stip_user: return stip_user, bean.instance if bean.tool: ret_tool = bean.tool stip_user = _get_match_stip_user(bean.tool) if stip_user: return stip_user, bean.tool if ret_instance: return None, ret_instance else: return None, ret_tool feed_user, screen_instance = _get_match_stip_user_from_bean(bean) if feed_user: feed.user = feed_user Feed.set_screen_value_from_local_db(feed, bean) feed.screen_instance = screen_instance else: feed.user = Feed.get_na_account() if screen_instance: feed.screen_name = screen_instance feed.screen_instance = screen_instance else: Feed.set_screen_value_from_local_db(feed, bean) feed.date = Feed.get_datetime_from_string(produced_str) feed.post_org = feed.post = stix_package.stix_header.description if not feed.post_org: feed.post_org = '' if not feed.post: feed.post = '' if Feed.is_stip_sns_stix_package_v1(stix_package): if stix_package.related_packages is not None: feed.save() for related_package in stix_package.related_packages: attach_file = Feed.get_attach_file( api_user, related_package.item.id_, 'v1') if attach_file is None: continue feed.files.add(attach_file) feed.save() feed.title = stix_package.stix_header.title feed.stix_file_path = stix_file_path try: uploader_stipuser = STIPUser.objects.get(id=uploader_id) feed.tlp = Feed.get_tlp_from_stix_header(stix_package.stix_header, uploader_stipuser.tlp) if feed.tlp is None: feed.tlp = uploader_stipuser.tlp except BaseException: feed.tlp = 'AMBER' sharing_range_info = Feed.get_sharing_range_from_stix_header( stix_package.stix_header) if isinstance(sharing_range_info, list): feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_PEOPLE feed.save() for stip_user in sharing_range_info: feed.sharing_people.add(stip_user) elif isinstance(sharing_range_info, Group): feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_GROUP feed.sharing_group = sharing_range_info else: feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_ALL if bean.region_code is not None: feed.region_code = bean.region_code else: if feed.user.region is not None: feed.region_code = feed.user.region.code else: feed.region_code = '' if bean.ci is not None: feed.ci = bean.ci else: feed.ci = feed.user.ci if bean.referred_url is not None: feed.referred_url = bean.referred_url if bean.stix2_package_id is not None: feed.stix2_package_id = bean.stix2_package_id feed.save() return feed