def _get_stix_content_dict(stix_file): if stix_file.version.startswith('2.') == True: #STIX 2.x return json.loads(stix_file.content.read()) else: #STIX 1.x stix_package = STIXPackage.from_xml(stix_file.origin_path) return stix_package.to_dict()
def update_stix_file_v1(pacakge_id, content): # 文字列イメージの引数のxmlをxml形式にして、インデントを揃えてダウンロードする file_path = get_file_name_from_package_id(pacakge_id) + '.xml' output = io.StringIO(content) # xml整形して再書き込み p = STIXPackage.from_xml(output) # 中身とファイル名を返却 return p.to_xml(), file_path
def get_attach_file(api_user, stix_id): attachment_stix_dir = Feed.get_attach_stix_dir_path(stix_id) if os.path.exists(attachment_stix_dir): # ここのファイル名とファイルパスを返却する try: # 1dir 1file なので最初の要素をファイル名とする attach_file = AttachFile() attach_file.file_name = Feed.get_attach_file_name(stix_id) attach_file.package_id = stix_id attach_file.save() return attach_file except IndexError: # dir はあるが fileが存在しない場合に発生する # 処理は続行する pass else: # dir が存在しないので作成 os.mkdir(attachment_stix_dir) attachement_cached_stix_file_path = rs.get_stix_file_path(api_user, stix_id) try: # Attachement STIX Package parse attachement_stix_package = STIXPackage.from_xml(attachement_cached_stix_file_path) # marking から file_name, content 取得 file_name = None content = None try: markings = attachement_stix_package.stix_header.handling.marking except AttributeError: return None for marking in markings: marking_structure = marking.marking_structures[0] if isinstance(marking_structure, SimpleMarkingStructure): statement = marking_structure.statement if statement.startswith(const.MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX): file_name = statement[len(const.MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX + ': '):] elif statement.startswith(const.MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX): content_str = statement[len(const.MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX + ': '):] # content は base64 で decode する content = base64.b64decode(content_str) if (file_name is None) or (content is None): return None except Exception as e: import traceback traceback.print_exc() raise e # ファイル保存 file_path = attachment_stix_dir + os.sep + file_name file_path = file_path.encode('utf-8') with open(file_path, 'wb') as fp: fp.write(content) attach_file = AttachFile() attach_file.file_name = file_name attach_file.package_id = stix_id attach_file.save() return attach_file
def update_stix_file_v1(pacakge_id, content): # 文字列イメージの引数のxmlをxml形式にして、インデントを揃えてダウンロードする file_path = get_file_name_from_package_id(pacakge_id) + '.xml' output = io.StringIO(content) # xml整形して再書き込み p = STIXPackage.from_xml(output) xml = p.to_xml() if isinstance(xml, bytes): xml = xml.decode() # 中身とファイル名を返却 return xml, file_path
def upload_xml(self, *vpath, **params): user = None try: user = self.user_controller.get_user_by_username( self.get_user().username) except ControllerException as error: self.logger.error(error) raise HTTPError(400, error.message) input_json = self.get_json() filename = input_json['name'] self.logger.info( 'Starting to import xml form file {0}'.format(filename)) data = input_json['data']['data'] complete = params.get('complete', False) inflated = params.get('inflated', False) xml_string = base64.b64decode(data) # STIX wants lxml instead of xml xml = etree.fromstring(xml_string) stix_package = STIXPackage.from_xml(xml) try: event = self.stix_mapper.map_stix_package(stix_package, user) event.properties.is_validated = False self.event_controller.insert_event(user, event, True, True) event_permissions = self.event_controller.get_event_user_permissions( event, user) cherrypy.response.headers[ 'Content-Type'] = 'application/json; charset=UTF-8' return json.dumps( event.to_dict(complete, inflated, event_permissions, user)) except ControllerIntegrityException as error: self.logger.debug(error) event = self.stix_mapper.map_stix_package(stix_package, user, False, True) local_event = self.event_controller.get_event_by_uuid(event.uuid) event_permissions = self.event_controller.get_event_user_permissions( event, user) merged_event = self.merger.merge_event(local_event, event, user, event_permissions) self.event_controller.update_event(user, merged_event, True, True) cherrypy.response.headers[ 'Content-Type'] = 'application/json; charset=UTF-8' return json.dumps( merged_event.to_dict(complete, inflated, event_permissions, user)) except ControllerNothingFoundException as error: self.logger.error(error) raise HTTPError(404, '{0}'.format(error.message)) except ControllerException as error: self.logger.error(error) raise HTTPError(400, '{0}'.format(error.message))
def upload_misp(self, package_id): stix_file = StixFiles.objects.get(package_id=package_id) stix_package = STIXPackage.from_xml(stix_file.content) misp_event = load_stix(stix_package) tag = self.get_tlp_tag(stix_package) if tag is not None: misp_event.add_tag(tag) resp = self.py_misp.add_event(misp_event) if resp.has_key('Event') == True: return resp else: raise Exception(str(resp['errors']))
def __init__(self, feed=None, stix_file_path=None, indicators=None, ttps=None, tas=None): super(FeedStix, self).__init__() self.stix_package = None self.attachment_files = [] if feed is not None: self.stix_package = self._make_stix_package( feed, indicators, ttps, tas) elif stix_file_path is not None: self.stix_package = STIXPackage.from_xml(stix_file_path)
def _get_event(self, uuid, version): payload = {} payload['uuid'] = uuid payload['returnFormat'] = version header = self.header.copy() if version == 'stix': header['Accept'] = 'application/xml' else: header['Accept'] = 'application/json' resp_text = self._post_misp(header, payload) if version == 'stix': with io.StringIO(resp_text) as fp: package = STIXPackage.from_xml(fp) return package else: return json.loads(resp_text)
def upload_misp(self, package_id): stix_file = StixFiles.objects.get(package_id=package_id) if stix_file.version.startswith('1.'): content = stix_file.content else: content = io.StringIO(stix_file.get_slide_12()) stix_package = STIXPackage.from_xml(content) misp_event = load_stix(stix_package) tag = self.get_tlp_tag(stix_package) if tag is not None: misp_event.add_tag(tag) resp = self.py_misp.add_event(misp_event) if ('Event' in resp): return resp else: raise Exception(str(resp['errors']))
def get_package_bean_v1(stix_file_path): doc = STIXPackage.from_xml(stix_file_path) try: package_bean = StixFiles.PackageBean() package_bean.is_post_sns = True package_bean.is_created_by_sns = False sns_type = None if _is_produced_by_stip_sns_v1(doc): package_bean.is_created_by_sns = True sns_type = _get_stip_sns_type_v1(doc) if sns_type != StixFiles.STIP_SNS_TYPE_ORIGIN: package_bean.is_post_sns = False try: package_bean.related_packages = [] for related_package in doc.related_packages: package_bean.related_packages.append(related_package.item.id_) except TypeError: package_bean.related_packages = None package_bean.package_id = doc.id_ package_bean.version = doc._version package_bean.produced = _get_produced_time_stix_1_x(doc) package_bean.package_name = doc.stix_header.title package_bean.sns_type = sns_type try: package_bean.description = doc.stix_header.description.value if package_bean.description is None: package_bean.description = '' except BaseException: package_bean.description = '' _set_stix_bean_from_doc_v1(package_bean, doc) if package_bean.sns_user_name == '': package_bean.sns_user_name = _get_sns_user_name_from_instance( package_bean.sns_instance) return package_bean except Exception: pass
def create_feeds_record(api_user, package_id, uploader_id, produced_str): # RS から取得した STIX から stix_package 取得する stix_file_path = rs.get_stix_file_path(api_user, package_id) stix_package = STIXPackage.from_xml(stix_file_path, encoding='utf-8') # Feed情報を STIX,RS の API から取得する feed = Feed() feed.package_id = package_id feed.filename_pk = rs.convert_package_id_to_filename(package_id) # STIX から表示情報を取得する bean = Feed.get_stip_from_stix_package(stix_package) if bean.is_sns: # SNS 産 STIX である if bean.instance == SNSConfig.get_sns_identity_name(): # 現在稼働中インスタンスと同一 try: # STIX 定義の username が存在する feed.user = STIPUser.objects.get(username=bean.user_name) # 表示はローカル DB から取得する Feed.set_screen_value_from_local_db(feed, bean) except BaseException: # STIX 定義の username が存在しない → N/A アカウント feed.user = Feed.get_na_account() # 表示はSTIX File から取得する Feed.set_screen_value_from_stix(feed, bean) # すでにユーザーが削除されている feed.is_valid_user = False else: # 現在稼働中インスタンスと異なる try: # インスタンス名と同じアカウント feed.user = STIPUser.objects.get(username=bean.instance) # 表示はSTIX File から取得する Feed.set_screen_value_from_stix(feed, bean) except BaseException: # インスタンス名と同じアカウントが存在しない → N/A アカウント feed.user = Feed.get_na_account() # 表示はSTIX File から取得する Feed.set_screen_value_from_stix(feed, bean) else: # SNS 産 STIX ではない if bean.instance is not None: # instance がある instance_user_name = bean.instance.replace(' ', '') try: # インスタンス名と同じアカウント feed.user = STIPUser.objects.get(username=instance_user_name) # 表示はローカル DB から取得する Feed.set_screen_value_from_local_db(feed, bean) except BaseException: # インスタンス名と同じアカウントが存在しない → N/A アカウント feed.user = Feed.get_na_account() # 表示は bean.instance [bean.instance] feed.screen_name = bean.instance feed.screen_instance = bean.instance else: # instance がない # N/A アカウント feed.user = Feed.get_na_account() # 表示はローカル DB(N/Aアカウント) から取得する Feed.set_screen_value_from_local_db(feed, bean) feed.date = Feed.get_datetime_from_string(produced_str) feed.post = stix_package.stix_header.description if feed.post is None: feed.post = '' # Attachement Files 情報取得 if Feed.is_stip_sns_stix_package(stix_package): # S-TIP SNS 作成 STIX if stix_package.related_packages is not None: # 一度 feed をsave()する feed.save() # Related_packages は SNS STIX 以外の可能性もある for related_package in stix_package.related_packages: # attachement は attachdirにいれるべきその時のファイル名は attachment_stix_idであるべき attach_file = Feed.get_attach_file(api_user, related_package.item.id_) # attach_file が None の場合は Attach File ではない if attach_file is None: continue feed.files.add(attach_file) feed.save() feed.title = stix_package.stix_header.title feed.stix_file_path = stix_file_path try: uploader_stipuser = STIPUser.objects.get(id=uploader_id) feed.tlp = Feed.get_ais_tlp_from_stix_header(stix_package.stix_header, uploader_stipuser.tlp) if feed.tlp is None: # 取得ができなかった場合は default TLP の AMBER feed.tlp = 'AMBER' except BaseException: # uploader_profile が存在しない場合は default TLP の AMBER feed.tlp = 'AMBER' sharing_range_info = Feed.get_sharing_range_from_stix_header(stix_package.stix_header) if isinstance(sharing_range_info, list): # sharing_range_info の中は STIPUser list feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_PEOPLE feed.save() for stip_user in sharing_range_info: feed.sharing_people.add(stip_user) elif isinstance(sharing_range_info, Group): feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_GROUP feed.sharing_group = sharing_range_info else: feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_ALL # feed.package_id = package_id if bean.region_code is not None: feed.region_code = bean.region_code else: if feed.user.region is not None: feed.region_code = feed.user.region.code else: feed.region_code = '' if bean.ci is not None: feed.ci = bean.ci else: feed.ci = feed.user.ci if bean.referred_url is not None: feed.referred_url = bean.referred_url if bean.stix2_package_id is not None: feed.stix2_package_id = bean.stix2_package_id feed.save() return feed
def get_package_bean(stix_file_path): package_bean = StixFiles.PackageBean() #STIX 1.1 parse try: doc = STIXPackage.from_xml(stix_file_path) package_bean.is_post_sns = True package_bean.is_created_by_sns = False sns_type = None #S-TIP SNS で作成された STIX であるか? if is_produced_by_stip_sns(doc) == True: #SNS 産である package_bean.is_created_by_sns = True sns_type = get_stip_sns_type(doc) #origin 投稿以外は表示しない if sns_type != StixFiles.STIP_SNS_TYPE_ORIGIN: package_bean.is_post_sns = False #realted_packages探す try: package_bean.related_packages = [] for related_package in doc.related_packages: package_bean.related_packages.append(related_package.item.id_) except TypeError: package_bean.related_packages = None package_bean.package_id = doc.id_ package_bean.version = doc._version package_bean.produced = get_produced_time_stix_1_x(doc) package_bean.package_name = doc.stix_header.title package_bean.sns_type = sns_type try: package_bean.description = doc.stix_header.description.value if package_bean.description is None: package_bean.description = '' except: package_bean.description = '' #S-TIP SNS 作成の STIX から pacakge_bean の値をセットする set_stix_bean_from_doc(package_bean, doc) #SNS 産以外は sns_user_name が設定されていないので instance 名から取得する if package_bean.sns_user_name == '': package_bean.sns_user_name = get_sns_user_name_from_instance( package_bean.sns_instance) return package_bean except Exception: pass #STIX 2.0 parse try: with codecs.open(stix_file_path, 'r', 'utf-8') as fp: content = fp.read() doc = json.loads(content) package_bean.package_name = None #最初に見つかったtypeがreportのnameをpackage_nameとする #また、STIX 2.0 では package の timestampの格納場所がないのでNoneとする produced_str = None for d in doc['objects']: if d['type'] == 'report': package_bean.package_name = d['name'] produced_str = d['created'] package_bean.package_id = doc['id'] if doc.has_key('spec_version') == True: package_bean.version = doc['spec_version'] else: #STIX 2.1 には spec_version がない package_bean.version = '2.1' #Produced Time は Report の produced time if produced_str is not None: package_bean.produced = stix2_str_to_datetime(produced_str) else: package_bean.produced = datetime.datetime.now() package_bean.is_post_sns = True package_bean.is_created_by_sns = False package_bean.related_packages = None return package_bean except Exception as e: traceback.print_exc() raise Exception('Can\'t parse STIX. ' + e.message)
def call_jira(request): try: # JIRA が import されていない場合は何もしない if imported_jira is None: rsp = {} return JsonResponse(rsp) # feed情報取得 feed_file_name_id = request.GET['feed_id'] package_id_arg = request.GET['package_id'] feed = Feed.get_feeds_from_package_id(request.user, package_id_arg) # JIRA instance proxies = System.get_request_proxies() j = JIRA(server=SNSConfig.get_jira_host(), proxies=proxies, basic_auth=(SNSConfig.get_jira_username(), SNSConfig.get_jira_password())) # issues作成 issue = j.create_issue(project=SNSConfig.get_jira_project(), summary=feed.title, description=feed.post, issuetype={'name': SNSConfig.get_jira_type()}) # 添付があればそれもつける for attach_file in feed.files.all(): file_path = Feed.get_attach_file_path(attach_file.package_id) j.add_attachment(issue=issue, attachment=file_path, filename=str(attach_file.file_name)) # STIX添付 stix_package = STIXPackage.from_xml(feed.stix_file_path) package_id = stix_package.id_ stix_file_name = '%s.xml' % (package_id) j.add_attachment(issue=issue, attachment=feed.stix_file_path, filename=stix_file_name) # CSV添付 # CSVの中身を取得する content = get_csv_content(feed_file_name_id) csv_attachment = io.StringIO() csv_attachment.write(content) csv_file_name = '%s.csv' % (package_id) j.add_attachment(issue=issue, attachment=csv_attachment, filename=csv_file_name) # PDF添付 feed_pdf = FeedPDF(feed, stix_package) pdf_attachment = io.BytesIO() feed_pdf.make_pdf_content(pdf_attachment, feed) pdf_file_name = '%s.pdf' % (package_id) j.add_attachment(issue=issue, attachment=pdf_attachment, filename=pdf_file_name) # isssue番号返却 url = SNSConfig.get_jira_host( ) + '/projects/' + SNSConfig.get_jira_project() + '/issues/' + str( issue) rsp = { 'issues': str(issue), 'url': url, } return JsonResponse(rsp) except Exception as e: traceback.print_exc() return HttpResponseServerError(str(e))
def create_feeds_record_v1(api_user, package_id, uploader_id, produced_str, version): stix_file_path = rs.get_stix_file_path(api_user, package_id) stix_package = STIXPackage.from_xml(stix_file_path) feed = Feed() feed.stix_version = version feed.package_id = package_id feed.filename_pk = rs.convert_package_id_to_filename(package_id) bean = Feed.get_stip_from_stix_package_v1(stix_package) if bean.is_sns: if bean.instance == SNSConfig.get_sns_identity_name(): try: feed.user = STIPUser.objects.get(username=bean.user_name) Feed.set_screen_value_from_local_db(feed, bean) except BaseException: feed.user = Feed.get_na_account() Feed.set_screen_value_from_stix(feed, bean) feed.is_valid_user = False else: try: feed.user = STIPUser.objects.get(username=bean.instance) Feed.set_screen_value_from_stix(feed, bean) except BaseException: feed.user = Feed.get_na_account() Feed.set_screen_value_from_stix(feed, bean) else: def _get_match_stip_user(username): try: username = username.replace(' ', '') return STIPUser.objects.get(username=username) except STIPUser.DoesNotExist: return None except Exception as e: raise e def _get_match_stip_user_from_bean(bean): ret_instance = None ret_tool = None if bean.instance: ret_instance = bean.instance stip_user = _get_match_stip_user(bean.instance) if stip_user: return stip_user, bean.instance if bean.tool: ret_tool = bean.tool stip_user = _get_match_stip_user(bean.tool) if stip_user: return stip_user, bean.tool if ret_instance: return None, ret_instance else: return None, ret_tool feed_user, screen_instance = _get_match_stip_user_from_bean(bean) if feed_user: feed.user = feed_user Feed.set_screen_value_from_local_db(feed, bean) feed.screen_instance = screen_instance else: feed.user = Feed.get_na_account() if screen_instance: feed.screen_name = screen_instance feed.screen_instance = screen_instance else: Feed.set_screen_value_from_local_db(feed, bean) feed.date = Feed.get_datetime_from_string(produced_str) feed.post_org = feed.post = stix_package.stix_header.description if not feed.post_org: feed.post_org = '' if not feed.post: feed.post = '' if Feed.is_stip_sns_stix_package_v1(stix_package): if stix_package.related_packages is not None: feed.save() for related_package in stix_package.related_packages: attach_file = Feed.get_attach_file( api_user, related_package.item.id_, 'v1') if attach_file is None: continue feed.files.add(attach_file) feed.save() feed.title = stix_package.stix_header.title feed.stix_file_path = stix_file_path try: uploader_stipuser = STIPUser.objects.get(id=uploader_id) feed.tlp = Feed.get_tlp_from_stix_header(stix_package.stix_header, uploader_stipuser.tlp) if feed.tlp is None: feed.tlp = uploader_stipuser.tlp except BaseException: feed.tlp = 'AMBER' sharing_range_info = Feed.get_sharing_range_from_stix_header( stix_package.stix_header) if isinstance(sharing_range_info, list): feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_PEOPLE feed.save() for stip_user in sharing_range_info: feed.sharing_people.add(stip_user) elif isinstance(sharing_range_info, Group): feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_GROUP feed.sharing_group = sharing_range_info else: feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_ALL if bean.region_code is not None: feed.region_code = bean.region_code else: if feed.user.region is not None: feed.region_code = feed.user.region.code else: feed.region_code = '' if bean.ci is not None: feed.ci = bean.ci else: feed.ci = feed.user.ci if bean.referred_url is not None: feed.referred_url = bean.referred_url if bean.stix2_package_id is not None: feed.stix2_package_id = bean.stix2_package_id feed.save() return feed
def get_attach_file(api_user, stix_id, version): attachment_stix_dir = Feed.get_attach_stix_dir_path(stix_id) if os.path.exists(attachment_stix_dir): try: attach_file = AttachFile() attach_file.file_name = Feed.get_attach_file_name(stix_id) attach_file.package_id = stix_id attach_file.save() return attach_file except IndexError: pass else: os.mkdir(attachment_stix_dir) attachement_cached_stix_file_path = rs.get_stix_file_path( api_user, stix_id) try: if version.startswith('v1'): attachement_stix_package = STIXPackage.from_xml( attachement_cached_stix_file_path) file_name = None content = None try: markings = attachement_stix_package.stix_header.handling.marking except AttributeError: return None for marking in markings: marking_structure = marking.marking_structures[0] if isinstance(marking_structure, SimpleMarkingStructure): statement = marking_structure.statement if statement.startswith( const. MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX ): file_name = statement[len( const. MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX + ': '):] elif statement.startswith( const. MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX ): content_str = statement[len( const. MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX + ': '):] content = base64.b64decode(content_str) if (file_name is None) or (content is None): return None elif version.startswith('v2'): with open(attachement_cached_stix_file_path, 'r', encoding='utf-8') as fp: attachment_bundle = json.load(fp) file_name = None content = None for o_ in attachment_bundle['objects']: if o_['type'] != const.STIP_STIX2_X_STIP_SNS_TYPE: continue if o_[const. STIP_STIX2_PROP_TYPE] != const.STIP_STIX2_SNS_POST_TYPE_ATTACHMENT: continue stip_sns_attachment = o_[const.STIP_STIX2_PROP_ATTACHMENT] content_str = stip_sns_attachment[ const.STIP_STIX2_SNS_ATTACHMENT_CONTENT_KEY] content = base64.b64decode(content_str) file_name = stip_sns_attachment[ const.STIP_STIX2_SNS_ATTACHMENT_FILENAME_KEY] break if (file_name is None) or (content is None): return None except Exception as e: import traceback traceback.print_exc() raise e file_path = attachment_stix_dir + os.sep + file_name file_path = file_path.encode('utf-8') with open(file_path, 'wb') as fp: fp.write(content) attach_file = AttachFile() attach_file.file_name = file_name attach_file.package_id = stix_id attach_file.save() return attach_file