def _get_stix_content_dict(stix_file):
if stix_file.version.startswith('2.') == True:
#STIX 2.x
return json.loads(stix_file.content.read())
else:
#STIX 1.x
stix_package = STIXPackage.from_xml(stix_file.origin_path)
return stix_package.to_dict()
def update_stix_file_v1(pacakge_id, content):
# 文字列イメージの引数のxmlをxml形式にして、インデントを揃えてダウンロードする
file_path = get_file_name_from_package_id(pacakge_id) + '.xml'
output = io.StringIO(content)
# xml整形して再書き込み
p = STIXPackage.from_xml(output)
# 中身とファイル名を返却
return p.to_xml(), file_path
def get_attach_file(api_user, stix_id):
attachment_stix_dir = Feed.get_attach_stix_dir_path(stix_id)
if os.path.exists(attachment_stix_dir):
# ここのファイル名とファイルパスを返却する
try:
# 1dir 1file なので最初の要素をファイル名とする
attach_file = AttachFile()
attach_file.file_name = Feed.get_attach_file_name(stix_id)
attach_file.package_id = stix_id
attach_file.save()
return attach_file
except IndexError:
# dir はあるが fileが存在しない場合に発生する
# 処理は続行する
pass
else:
# dir が存在しないので作成
os.mkdir(attachment_stix_dir)
attachement_cached_stix_file_path = rs.get_stix_file_path(api_user, stix_id)
try:
# Attachement STIX Package parse
attachement_stix_package = STIXPackage.from_xml(attachement_cached_stix_file_path)
# marking から file_name, content 取得
file_name = None
content = None
try:
markings = attachement_stix_package.stix_header.handling.marking
except AttributeError:
return None
for marking in markings:
marking_structure = marking.marking_structures[0]
if isinstance(marking_structure, SimpleMarkingStructure):
statement = marking_structure.statement
if statement.startswith(const.MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX):
file_name = statement[len(const.MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX + ': '):]
elif statement.startswith(const.MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX):
content_str = statement[len(const.MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX + ': '):]
# content は base64 で decode する
content = base64.b64decode(content_str)
if (file_name is None) or (content is None):
return None
except Exception as e:
import traceback
traceback.print_exc()
raise e
# ファイル保存
file_path = attachment_stix_dir + os.sep + file_name
file_path = file_path.encode('utf-8')
with open(file_path, 'wb') as fp:
fp.write(content)
attach_file = AttachFile()
attach_file.file_name = file_name
attach_file.package_id = stix_id
attach_file.save()
return attach_file
def update_stix_file_v1(pacakge_id, content):
# 文字列イメージの引数のxmlをxml形式にして、インデントを揃えてダウンロードする
file_path = get_file_name_from_package_id(pacakge_id) + '.xml'
output = io.StringIO(content)
# xml整形して再書き込み
p = STIXPackage.from_xml(output)
xml = p.to_xml()
if isinstance(xml, bytes):
xml = xml.decode()
# 中身とファイル名を返却
return xml, file_path
def upload_xml(self, *vpath, **params):
user = None
try:
user = self.user_controller.get_user_by_username(
self.get_user().username)
except ControllerException as error:
self.logger.error(error)
raise HTTPError(400, error.message)
input_json = self.get_json()
filename = input_json['name']
self.logger.info(
'Starting to import xml form file {0}'.format(filename))
data = input_json['data']['data']
complete = params.get('complete', False)
inflated = params.get('inflated', False)
xml_string = base64.b64decode(data)
# STIX wants lxml instead of xml
xml = etree.fromstring(xml_string)
stix_package = STIXPackage.from_xml(xml)
try:
event = self.stix_mapper.map_stix_package(stix_package, user)
event.properties.is_validated = False
self.event_controller.insert_event(user, event, True, True)
event_permissions = self.event_controller.get_event_user_permissions(
event, user)
cherrypy.response.headers[
'Content-Type'] = 'application/json; charset=UTF-8'
return json.dumps(
event.to_dict(complete, inflated, event_permissions, user))
except ControllerIntegrityException as error:
self.logger.debug(error)
event = self.stix_mapper.map_stix_package(stix_package, user,
False, True)
local_event = self.event_controller.get_event_by_uuid(event.uuid)
event_permissions = self.event_controller.get_event_user_permissions(
event, user)
merged_event = self.merger.merge_event(local_event, event, user,
event_permissions)
self.event_controller.update_event(user, merged_event, True, True)
cherrypy.response.headers[
'Content-Type'] = 'application/json; charset=UTF-8'
return json.dumps(
merged_event.to_dict(complete, inflated, event_permissions,
user))
except ControllerNothingFoundException as error:
self.logger.error(error)
raise HTTPError(404, '{0}'.format(error.message))
except ControllerException as error:
self.logger.error(error)
raise HTTPError(400, '{0}'.format(error.message))
def upload_misp(self, package_id):
stix_file = StixFiles.objects.get(package_id=package_id)
stix_package = STIXPackage.from_xml(stix_file.content)
misp_event = load_stix(stix_package)
tag = self.get_tlp_tag(stix_package)
if tag is not None:
misp_event.add_tag(tag)
resp = self.py_misp.add_event(misp_event)
if resp.has_key('Event') == True:
return resp
else:
raise Exception(str(resp['errors']))
def __init__(self,
feed=None,
stix_file_path=None,
indicators=None,
ttps=None,
tas=None):
super(FeedStix, self).__init__()
self.stix_package = None
self.attachment_files = []
if feed is not None:
self.stix_package = self._make_stix_package(
feed, indicators, ttps, tas)
elif stix_file_path is not None:
self.stix_package = STIXPackage.from_xml(stix_file_path)
def _get_event(self, uuid, version):
payload = {}
payload['uuid'] = uuid
payload['returnFormat'] = version
header = self.header.copy()
if version == 'stix':
header['Accept'] = 'application/xml'
else:
header['Accept'] = 'application/json'
resp_text = self._post_misp(header, payload)
if version == 'stix':
with io.StringIO(resp_text) as fp:
package = STIXPackage.from_xml(fp)
return package
else:
return json.loads(resp_text)
def upload_misp(self, package_id):
stix_file = StixFiles.objects.get(package_id=package_id)
if stix_file.version.startswith('1.'):
content = stix_file.content
else:
content = io.StringIO(stix_file.get_slide_12())
stix_package = STIXPackage.from_xml(content)
misp_event = load_stix(stix_package)
tag = self.get_tlp_tag(stix_package)
if tag is not None:
misp_event.add_tag(tag)
resp = self.py_misp.add_event(misp_event)
if ('Event' in resp):
return resp
else:
raise Exception(str(resp['errors']))
def get_package_bean_v1(stix_file_path):
doc = STIXPackage.from_xml(stix_file_path)
try:
package_bean = StixFiles.PackageBean()
package_bean.is_post_sns = True
package_bean.is_created_by_sns = False
sns_type = None
if _is_produced_by_stip_sns_v1(doc):
package_bean.is_created_by_sns = True
sns_type = _get_stip_sns_type_v1(doc)
if sns_type != StixFiles.STIP_SNS_TYPE_ORIGIN:
package_bean.is_post_sns = False
try:
package_bean.related_packages = []
for related_package in doc.related_packages:
package_bean.related_packages.append(related_package.item.id_)
except TypeError:
package_bean.related_packages = None
package_bean.package_id = doc.id_
package_bean.version = doc._version
package_bean.produced = _get_produced_time_stix_1_x(doc)
package_bean.package_name = doc.stix_header.title
package_bean.sns_type = sns_type
try:
package_bean.description = doc.stix_header.description.value
if package_bean.description is None:
package_bean.description = ''
except BaseException:
package_bean.description = ''
_set_stix_bean_from_doc_v1(package_bean, doc)
if package_bean.sns_user_name == '':
package_bean.sns_user_name = _get_sns_user_name_from_instance(
package_bean.sns_instance)
return package_bean
except Exception:
pass
def create_feeds_record(api_user, package_id, uploader_id, produced_str):
# RS から取得した STIX から stix_package 取得する
stix_file_path = rs.get_stix_file_path(api_user, package_id)
stix_package = STIXPackage.from_xml(stix_file_path, encoding='utf-8')
# Feed情報を STIX,RS の API から取得する
feed = Feed()
feed.package_id = package_id
feed.filename_pk = rs.convert_package_id_to_filename(package_id)
# STIX から表示情報を取得する
bean = Feed.get_stip_from_stix_package(stix_package)
if bean.is_sns:
# SNS 産 STIX である
if bean.instance == SNSConfig.get_sns_identity_name():
# 現在稼働中インスタンスと同一
try:
# STIX 定義の username が存在する
feed.user = STIPUser.objects.get(username=bean.user_name)
# 表示はローカル DB から取得する
Feed.set_screen_value_from_local_db(feed, bean)
except BaseException:
# STIX 定義の username が存在しない → N/A アカウント
feed.user = Feed.get_na_account()
# 表示はSTIX File から取得する
Feed.set_screen_value_from_stix(feed, bean)
# すでにユーザーが削除されている
feed.is_valid_user = False
else:
# 現在稼働中インスタンスと異なる
try:
# インスタンス名と同じアカウント
feed.user = STIPUser.objects.get(username=bean.instance)
# 表示はSTIX File から取得する
Feed.set_screen_value_from_stix(feed, bean)
except BaseException:
# インスタンス名と同じアカウントが存在しない → N/A アカウント
feed.user = Feed.get_na_account()
# 表示はSTIX File から取得する
Feed.set_screen_value_from_stix(feed, bean)
else:
# SNS 産 STIX ではない
if bean.instance is not None:
# instance がある
instance_user_name = bean.instance.replace(' ', '')
try:
# インスタンス名と同じアカウント
feed.user = STIPUser.objects.get(username=instance_user_name)
# 表示はローカル DB から取得する
Feed.set_screen_value_from_local_db(feed, bean)
except BaseException:
# インスタンス名と同じアカウントが存在しない → N/A アカウント
feed.user = Feed.get_na_account()
# 表示は bean.instance [bean.instance]
feed.screen_name = bean.instance
feed.screen_instance = bean.instance
else:
# instance がない
# N/A アカウント
feed.user = Feed.get_na_account()
# 表示はローカル DB(N/Aアカウント) から取得する
Feed.set_screen_value_from_local_db(feed, bean)
feed.date = Feed.get_datetime_from_string(produced_str)
feed.post = stix_package.stix_header.description
if feed.post is None:
feed.post = ''
# Attachement Files 情報取得
if Feed.is_stip_sns_stix_package(stix_package):
# S-TIP SNS 作成 STIX
if stix_package.related_packages is not None:
# 一度 feed をsave()する
feed.save()
# Related_packages は SNS STIX 以外の可能性もある
for related_package in stix_package.related_packages:
# attachement は attachdirにいれるべきその時のファイル名は attachment_stix_idであるべき
attach_file = Feed.get_attach_file(api_user, related_package.item.id_)
# attach_file が None の場合は Attach File ではない
if attach_file is None:
continue
feed.files.add(attach_file)
feed.save()
feed.title = stix_package.stix_header.title
feed.stix_file_path = stix_file_path
try:
uploader_stipuser = STIPUser.objects.get(id=uploader_id)
feed.tlp = Feed.get_ais_tlp_from_stix_header(stix_package.stix_header, uploader_stipuser.tlp)
if feed.tlp is None:
# 取得ができなかった場合は default TLP の AMBER
feed.tlp = 'AMBER'
except BaseException:
# uploader_profile が存在しない場合は default TLP の AMBER
feed.tlp = 'AMBER'
sharing_range_info = Feed.get_sharing_range_from_stix_header(stix_package.stix_header)
if isinstance(sharing_range_info, list):
# sharing_range_info の中は STIPUser list
feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_PEOPLE
feed.save()
for stip_user in sharing_range_info:
feed.sharing_people.add(stip_user)
elif isinstance(sharing_range_info, Group):
feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_GROUP
feed.sharing_group = sharing_range_info
else:
feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_ALL
# feed.package_id = package_id
if bean.region_code is not None:
feed.region_code = bean.region_code
else:
if feed.user.region is not None:
feed.region_code = feed.user.region.code
else:
feed.region_code = ''
if bean.ci is not None:
feed.ci = bean.ci
else:
feed.ci = feed.user.ci
if bean.referred_url is not None:
feed.referred_url = bean.referred_url
if bean.stix2_package_id is not None:
feed.stix2_package_id = bean.stix2_package_id
feed.save()
return feed
def get_package_bean(stix_file_path):
package_bean = StixFiles.PackageBean()
#STIX 1.1 parse
try:
doc = STIXPackage.from_xml(stix_file_path)
package_bean.is_post_sns = True
package_bean.is_created_by_sns = False
sns_type = None
#S-TIP SNS で作成された STIX であるか?
if is_produced_by_stip_sns(doc) == True:
#SNS 産である
package_bean.is_created_by_sns = True
sns_type = get_stip_sns_type(doc)
#origin 投稿以外は表示しない
if sns_type != StixFiles.STIP_SNS_TYPE_ORIGIN:
package_bean.is_post_sns = False
#realted_packages探す
try:
package_bean.related_packages = []
for related_package in doc.related_packages:
package_bean.related_packages.append(related_package.item.id_)
except TypeError:
package_bean.related_packages = None
package_bean.package_id = doc.id_
package_bean.version = doc._version
package_bean.produced = get_produced_time_stix_1_x(doc)
package_bean.package_name = doc.stix_header.title
package_bean.sns_type = sns_type
try:
package_bean.description = doc.stix_header.description.value
if package_bean.description is None:
package_bean.description = ''
except:
package_bean.description = ''
#S-TIP SNS 作成の STIX から pacakge_bean の値をセットする
set_stix_bean_from_doc(package_bean, doc)
#SNS 産以外は sns_user_name が設定されていないので instance 名から取得する
if package_bean.sns_user_name == '':
package_bean.sns_user_name = get_sns_user_name_from_instance(
package_bean.sns_instance)
return package_bean
except Exception:
pass
#STIX 2.0 parse
try:
with codecs.open(stix_file_path, 'r', 'utf-8') as fp:
content = fp.read()
doc = json.loads(content)
package_bean.package_name = None
#最初に見つかったtypeがreportのnameをpackage_nameとする
#また、STIX 2.0 では package の timestampの格納場所がないのでNoneとする
produced_str = None
for d in doc['objects']:
if d['type'] == 'report':
package_bean.package_name = d['name']
produced_str = d['created']
package_bean.package_id = doc['id']
if doc.has_key('spec_version') == True:
package_bean.version = doc['spec_version']
else:
#STIX 2.1 には spec_version がない
package_bean.version = '2.1'
#Produced Time は Report の produced time
if produced_str is not None:
package_bean.produced = stix2_str_to_datetime(produced_str)
else:
package_bean.produced = datetime.datetime.now()
package_bean.is_post_sns = True
package_bean.is_created_by_sns = False
package_bean.related_packages = None
return package_bean
except Exception as e:
traceback.print_exc()
raise Exception('Can\'t parse STIX. ' + e.message)
def call_jira(request):
try:
# JIRA が import されていない場合は何もしない
if imported_jira is None:
rsp = {}
return JsonResponse(rsp)
# feed情報取得
feed_file_name_id = request.GET['feed_id']
package_id_arg = request.GET['package_id']
feed = Feed.get_feeds_from_package_id(request.user, package_id_arg)
# JIRA instance
proxies = System.get_request_proxies()
j = JIRA(server=SNSConfig.get_jira_host(),
proxies=proxies,
basic_auth=(SNSConfig.get_jira_username(),
SNSConfig.get_jira_password()))
# issues作成
issue = j.create_issue(project=SNSConfig.get_jira_project(),
summary=feed.title,
description=feed.post,
issuetype={'name': SNSConfig.get_jira_type()})
# 添付があればそれもつける
for attach_file in feed.files.all():
file_path = Feed.get_attach_file_path(attach_file.package_id)
j.add_attachment(issue=issue,
attachment=file_path,
filename=str(attach_file.file_name))
# STIX添付
stix_package = STIXPackage.from_xml(feed.stix_file_path)
package_id = stix_package.id_
stix_file_name = '%s.xml' % (package_id)
j.add_attachment(issue=issue,
attachment=feed.stix_file_path,
filename=stix_file_name)
# CSV添付
# CSVの中身を取得する
content = get_csv_content(feed_file_name_id)
csv_attachment = io.StringIO()
csv_attachment.write(content)
csv_file_name = '%s.csv' % (package_id)
j.add_attachment(issue=issue,
attachment=csv_attachment,
filename=csv_file_name)
# PDF添付
feed_pdf = FeedPDF(feed, stix_package)
pdf_attachment = io.BytesIO()
feed_pdf.make_pdf_content(pdf_attachment, feed)
pdf_file_name = '%s.pdf' % (package_id)
j.add_attachment(issue=issue,
attachment=pdf_attachment,
filename=pdf_file_name)
# isssue番号返却
url = SNSConfig.get_jira_host(
) + '/projects/' + SNSConfig.get_jira_project() + '/issues/' + str(
issue)
rsp = {
'issues': str(issue),
'url': url,
}
return JsonResponse(rsp)
except Exception as e:
traceback.print_exc()
return HttpResponseServerError(str(e))
def create_feeds_record_v1(api_user, package_id, uploader_id, produced_str,
version):
stix_file_path = rs.get_stix_file_path(api_user, package_id)
stix_package = STIXPackage.from_xml(stix_file_path)
feed = Feed()
feed.stix_version = version
feed.package_id = package_id
feed.filename_pk = rs.convert_package_id_to_filename(package_id)
bean = Feed.get_stip_from_stix_package_v1(stix_package)
if bean.is_sns:
if bean.instance == SNSConfig.get_sns_identity_name():
try:
feed.user = STIPUser.objects.get(username=bean.user_name)
Feed.set_screen_value_from_local_db(feed, bean)
except BaseException:
feed.user = Feed.get_na_account()
Feed.set_screen_value_from_stix(feed, bean)
feed.is_valid_user = False
else:
try:
feed.user = STIPUser.objects.get(username=bean.instance)
Feed.set_screen_value_from_stix(feed, bean)
except BaseException:
feed.user = Feed.get_na_account()
Feed.set_screen_value_from_stix(feed, bean)
else:
def _get_match_stip_user(username):
try:
username = username.replace(' ', '')
return STIPUser.objects.get(username=username)
except STIPUser.DoesNotExist:
return None
except Exception as e:
raise e
def _get_match_stip_user_from_bean(bean):
ret_instance = None
ret_tool = None
if bean.instance:
ret_instance = bean.instance
stip_user = _get_match_stip_user(bean.instance)
if stip_user:
return stip_user, bean.instance
if bean.tool:
ret_tool = bean.tool
stip_user = _get_match_stip_user(bean.tool)
if stip_user:
return stip_user, bean.tool
if ret_instance:
return None, ret_instance
else:
return None, ret_tool
feed_user, screen_instance = _get_match_stip_user_from_bean(bean)
if feed_user:
feed.user = feed_user
Feed.set_screen_value_from_local_db(feed, bean)
feed.screen_instance = screen_instance
else:
feed.user = Feed.get_na_account()
if screen_instance:
feed.screen_name = screen_instance
feed.screen_instance = screen_instance
else:
Feed.set_screen_value_from_local_db(feed, bean)
feed.date = Feed.get_datetime_from_string(produced_str)
feed.post_org = feed.post = stix_package.stix_header.description
if not feed.post_org:
feed.post_org = ''
if not feed.post:
feed.post = ''
if Feed.is_stip_sns_stix_package_v1(stix_package):
if stix_package.related_packages is not None:
feed.save()
for related_package in stix_package.related_packages:
attach_file = Feed.get_attach_file(
api_user, related_package.item.id_, 'v1')
if attach_file is None:
continue
feed.files.add(attach_file)
feed.save()
feed.title = stix_package.stix_header.title
feed.stix_file_path = stix_file_path
try:
uploader_stipuser = STIPUser.objects.get(id=uploader_id)
feed.tlp = Feed.get_tlp_from_stix_header(stix_package.stix_header,
uploader_stipuser.tlp)
if feed.tlp is None:
feed.tlp = uploader_stipuser.tlp
except BaseException:
feed.tlp = 'AMBER'
sharing_range_info = Feed.get_sharing_range_from_stix_header(
stix_package.stix_header)
if isinstance(sharing_range_info, list):
feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_PEOPLE
feed.save()
for stip_user in sharing_range_info:
feed.sharing_people.add(stip_user)
elif isinstance(sharing_range_info, Group):
feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_GROUP
feed.sharing_group = sharing_range_info
else:
feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_ALL
if bean.region_code is not None:
feed.region_code = bean.region_code
else:
if feed.user.region is not None:
feed.region_code = feed.user.region.code
else:
feed.region_code = ''
if bean.ci is not None:
feed.ci = bean.ci
else:
feed.ci = feed.user.ci
if bean.referred_url is not None:
feed.referred_url = bean.referred_url
if bean.stix2_package_id is not None:
feed.stix2_package_id = bean.stix2_package_id
feed.save()
return feed
def get_attach_file(api_user, stix_id, version):
attachment_stix_dir = Feed.get_attach_stix_dir_path(stix_id)
if os.path.exists(attachment_stix_dir):
try:
attach_file = AttachFile()
attach_file.file_name = Feed.get_attach_file_name(stix_id)
attach_file.package_id = stix_id
attach_file.save()
return attach_file
except IndexError:
pass
else:
os.mkdir(attachment_stix_dir)
attachement_cached_stix_file_path = rs.get_stix_file_path(
api_user, stix_id)
try:
if version.startswith('v1'):
attachement_stix_package = STIXPackage.from_xml(
attachement_cached_stix_file_path)
file_name = None
content = None
try:
markings = attachement_stix_package.stix_header.handling.marking
except AttributeError:
return None
for marking in markings:
marking_structure = marking.marking_structures[0]
if isinstance(marking_structure, SimpleMarkingStructure):
statement = marking_structure.statement
if statement.startswith(
const.
MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX
):
file_name = statement[len(
const.
MARKING_STRUCTURE_STIP_ATTACHEMENT_FILENAME_PREFIX
+ ': '):]
elif statement.startswith(
const.
MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX
):
content_str = statement[len(
const.
MARKING_STRUCTURE_STIP_ATTACHEMENT_CONTENT_PREFIX
+ ': '):]
content = base64.b64decode(content_str)
if (file_name is None) or (content is None):
return None
elif version.startswith('v2'):
with open(attachement_cached_stix_file_path,
'r',
encoding='utf-8') as fp:
attachment_bundle = json.load(fp)
file_name = None
content = None
for o_ in attachment_bundle['objects']:
if o_['type'] != const.STIP_STIX2_X_STIP_SNS_TYPE:
continue
if o_[const.
STIP_STIX2_PROP_TYPE] != const.STIP_STIX2_SNS_POST_TYPE_ATTACHMENT:
continue
stip_sns_attachment = o_[const.STIP_STIX2_PROP_ATTACHMENT]
content_str = stip_sns_attachment[
const.STIP_STIX2_SNS_ATTACHMENT_CONTENT_KEY]
content = base64.b64decode(content_str)
file_name = stip_sns_attachment[
const.STIP_STIX2_SNS_ATTACHMENT_FILENAME_KEY]
break
if (file_name is None) or (content is None):
return None
except Exception as e:
import traceback
traceback.print_exc()
raise e
file_path = attachment_stix_dir + os.sep + file_name
file_path = file_path.encode('utf-8')
with open(file_path, 'wb') as fp:
fp.write(content)
attach_file = AttachFile()
attach_file.file_name = file_name
attach_file.package_id = stix_id
attach_file.save()
return attach_file