def set_extractor_info(stip_params, attached_files, username):
try:
stip_user = STIPUser.objects.get(username=username)
except BaseException:
stip_user = None
if stip_user is not None:
white_list = get_merged_conf_list(
SNSConfig.get_common_white_list(),
stip_user.sns_profile.indicator_white_list)
ta_list = get_merged_conf_list(SNSConfig.get_common_ta_list(),
stip_user.sns_profile.threat_actors)
else:
ta_list = []
white_list = []
confirm_indicators, confirm_ets, confirm_tas = Extractor.get_stix_element(
files=attached_files,
posts=[stip_params[STIP_PARAMS_INDEX_POST]],
referred_url=stip_params[STIP_PARAMS_INDEX_REFERRED_URL]
if len(stip_params[STIP_PARAMS_INDEX_REFERRED_URL]) != 0 else None,
ta_list=ta_list,
white_list=white_list)
stip_params[STIP_PARAMS_INDEX_INDICATORS] = json.dumps(
get_extractor_items(confirm_indicators))
stip_params[STIP_PARAMS_INDEX_TTPS] = json.dumps(
get_extractor_items(confirm_ets))
stip_params[STIP_PARAMS_INDEX_TAS] = json.dumps(
get_extractor_items(confirm_tas))
return stip_params
def _get_threat_actor_object(ta, stip_identity, tlp_marking_object):
name = ta['value']
description = ta['title']
try:
threat_actor_types = [ta['type']]
except KeyError:
threat_actor_types = ['unknown']
if SNSConfig.get_cs_custid() and SNSConfig.get_cs_custkey():
description, aliases = fec.CommonExtractor._get_ta_description_from_crowd_strike(
name)
if not description:
description, aliases = fec.CommonExtractor._get_ta_description_from_attck(
name)
else:
description, aliases = fec.CommonExtractor._get_ta_description_from_attck(
name)
threat_actor = ThreatActor(name=name,
description=description,
created_by_ref=stip_identity,
object_marking_refs=[tlp_marking_object],
aliases=aliases,
threat_actor_types=threat_actor_types)
return threat_actor
def __init__(self):
# namespace dictionary
self.ns_dict = {
SNSConfig.get_stix_ns_url(): SNSConfig.get_stix_ns_name(),
}
ns_ctim_sns = Namespace(SNSConfig.get_stix_ns_url(), SNSConfig.get_stix_ns_name(), schema_location=None)
# id generator
idgen.set_id_namespace(ns_ctim_sns)
self.generator = idgen._get_generator()
def headers(request):
d = {}
# SNS Header Title
d['sns_base_header_title'] = SNSConfig.get_sns_header_title()
# BodyColor
d['sns_base_body_color'] = SNSConfig.get_sns_body_color()
# GitVersion
d['git_version'] = SNSConfig.get_sns_version()
return d
def get_access_token(proxies):
OAUTH2_URL = BASE_URL + '/oauth2/token'
payload = {}
payload['client_id'] = SNSConfig.get_cs_custid()
payload['client_secret'] = SNSConfig.get_cs_custkey()
headers = get_crowd_strike_request_header()
return requests.post(
url=OAUTH2_URL,
params=headers,
data=payload,
proxies=proxies)
def _make_stix_package(self, feed, indicators=[], ttps=[], tas=[]):
user_timezone = pytz.timezone(feed.user.timezone)
# package ID作成
package_id = self.generator.create_id(prefix='Package')
# package作成
stix_package = STIXPackage(id_=package_id)
stix_package.timestamp = datetime.datetime.now(tz=user_timezone)
# header格納
stix_package.stix_header = self._get_stix_header(feed)
# indicators 格納
# web 画面から取得した indicators (json) から stix indicators 作成する
stix_indicators = Indicators()
for indicator_json in indicators:
indicator = CommonExtractor.get_indicator_from_json(
indicator_json, user_timezone)
if indicator is not None:
stix_indicators.append(indicator)
stix_package.indicators = stix_indicators
# ExploitTargets格納
stix_exploit_targets = ExploitTargets()
for ttp_json in ttps:
et = CommonExtractor.get_exploit_target_from_json(ttp_json)
if et is not None:
stix_exploit_targets.append(et)
stix_package.exploit_targets = stix_exploit_targets
# ThreatActors 格納
for ta_json in tas:
value = ta_json['value']
if SNSConfig.get_cs_custid(
) is not None and SNSConfig.get_cs_custkey() is not None:
ta = self.get_ta_from_crowd_strike(value)
if ta is None:
# ATT&CK から ThreatActor 取得する
ta = self.get_ta_from_attck(value)
else:
ta = self.get_ta_from_attck(value)
stix_package.add_threat_actor(ta)
# 添付ファイル用の STIX 作成する
for file_ in feed.files.all():
attach_file_stix_package = self._make_stix_package_for_attached_file(
file_, feed)
self.attachment_files.append(attach_file_stix_package)
# 添付ファイル用の STIX を Related Pacakge に追加する
stix_package.add_related_package(attach_file_stix_package.id_)
return stix_package
def get_crowd_strike_request_header():
query_http_headers = {}
custid = SNSConfig.get_cs_custid()
custkey = SNSConfig.get_cs_custkey()
if custid is not None:
query_http_headers['X-CSIX-CUSTID'] = custid
else:
raise Exception('No X-CSIX-CUSTID configuration.')
if custkey is not None:
query_http_headers['X-CSIX-CUSTKEY'] = custkey
else:
raise Exception('No X-CSIX-CUSTKEY configuration.')
query_http_headers['Content-Type'] = 'application/json'
return query_http_headers
def _get_ctim_gv_url(request):
# 設定ファイルに指定があったらその値を使う
if (SNSConfig.get_gv_l2_url()
is not None) and (len(SNSConfig.get_gv_l2_url()) != 0):
return SNSConfig.get_gv_l2_url()
# ない場合は request の値から URL 構築する
scheme = request.scheme
host = request.get_host()
host_split = host.split(':')
if len(host_split) == 1:
gv_host = '%s:%d' % (host, DEFAULT_GV_PORT)
else:
gv_host = '%s:%d' % (host_split[0], DEFAULT_GV_PORT)
gv_url = '%s://%s%s' % (scheme, gv_host, L2_GV_PATH)
return gv_url
def start_receive_slack_thread():
global slack_token
global wc
global th
global rtm_client
slack_token = SNSConfig.get_slack_bot_token()
if slack_token is None:
print('Slack token is undefined.')
return
if len(slack_token) == 0:
print('Slack token length is 0.')
return
# Slack ユーザがいなければ作る
slack_users = STIPUser.objects.filter(username=SNS_SLACK_BOT_ACCOUNT)
if len(slack_users) == 0:
# slack ユーザ作成する
slack_user = STIPUser.objects.create_user(
SNS_SLACK_BOT_ACCOUNT,
SNS_SLACK_BOT_ACCOUNT,
SNS_SLACK_BOT_ACCOUNT,
is_admin=False)
slack_user.save()
wc = slack.WebClient(token=slack_token)
rtm_client = slack.RTMClient(token=slack_token)
th = SlackThread()
return
def like(request):
# Like元のパッケージID
package_id = request.POST['package_id']
feed = Feed.get_feeds_from_package_id(request.user, package_id)
# user は STIPUser
stip_user = request.user
# liker情報取得
likers = rs.get_likers_from_rs(stip_user, package_id)
# すでにLikeされているか判定
# 自分自身の liker 文字列は instance_name + space + user_name
myliker = '%s %s' % (SNSConfig.get_sns_identity_name(), stip_user.username)
like = myliker in likers
# Like/Unlike 用の STIX イメージ作成
feed_stix_like = FeedStixLike(feed, like, creator=stip_user)
if like:
# notify の unlike処理
stip_user.unotify_liked(package_id, feed.user)
else:
# notify の like処理
stip_user.notify_liked(package_id, feed.user)
# 一時ファイルにstixの中身を書き出す
tmp_file_path = write_like_comment_attach_stix(
feed_stix_like.get_xml_content())
# RS に登録する
rs.regist_ctim_rs(stip_user, feed_stix_like.file_name, tmp_file_path)
os.remove(tmp_file_path)
# 現在の Like 情報を取得する
likers = rs.get_likers_from_rs(stip_user, package_id)
return HttpResponse(len(likers))
def get_attached_file_from_slack(file_path):
slack_token = SNSConfig.get_slack_bot_token()
headers = {}
headers['Authorization'] = 'Bearer ' + slack_token
proxies = System.get_request_proxies()
resp = requests.get(url=file_path, headers=headers, proxies=proxies)
return resp
def get_feeds_from_package_from_rs(api_user, package_from_rs):
package_id = package_from_rs['package_id']
uploader_id = package_from_rs['uploader']
produced_str = package_from_rs['produced']
try:
# cache にあれば採用する
feed = Feed.objects.get(package_id=package_id)
# STIX の instance がこの稼働している instance と同じであるかチェック
if feed.screen_instance is not None:
if feed.screen_instance == SNSConfig.get_sns_identity_name():
# feed.user の現在の affiliation/screen_name/ci/region_codeを使用する
feed.screen_name = feed.user.screen_name
feed.screen_affiliation = feed.user.affiliation
feed.ci = feed.user.ci
if feed.user.region is not None:
feed.region_code = feed.user.region.code
except Feed.DoesNotExist as e:
# cache 作成
feed = Feed.create_feeds_record(api_user, package_id, uploader_id, produced_str)
except Exception as e:
import traceback
traceback.print_exc()
raise e
return feed
def add_like_comment_info(api_user, feed):
likers = rs.get_likers_from_rs(api_user, feed.package_id)
feed.likes = len(likers)
mylike = '%s %s' % (SNSConfig.get_sns_identity_name(), api_user)
feed.like = mylike in likers
feed.comments = len(rs.get_comment_from_rs(api_user, feed.package_id))
return feed
def receive_slack(**payload):
# channel 名から先頭の # を外す
slack_bot_channel_name = SNSConfig.get_slack_bot_chnnel()[1:]
slack_user = STIPUser.objects.get(username=SNS_SLACK_BOT_ACCOUNT)
receive_data = payload['data']
try:
post_stip_from_slack(receive_data, slack_bot_channel_name, slack_user)
except BaseException:
import traceback
traceback.print_exc()
def download_stix_id(command_stix_id):
wc = StipSnsBoot.get_slack_web_client()
# cache の STIX を返却
stix_file_path = Feed.get_cached_file_path(
command_stix_id.replace(':', '--'))
file_name = '%s.xml' % (command_stix_id)
post_slack_channel = SNSConfig.get_slack_bot_chnnel()
wc.files_upload(initial_comment='',
channels=post_slack_channel,
file=open(stix_file_path, 'rb'),
filename=file_name)
return
def init_mongo():
# 接続する
try:
connect(SNSConfig.get_circl_mongo_database(),
host=SNSConfig.get_circl_mongo_host(),
port=SNSConfig.get_circl_mongo_port(),
alias='circl')
except BaseException:
# エラーの場合はデフォルト設定
connect(SNSConfig.DEFAULT_CIRCL_MONGO_DATABASE,
host=SNSConfig.DEFAULT_CIRCL_MONGO_HOST,
port=SNSConfig.DEFAULT_CIRCL_MONGO_PORT,
alias='circl')
try:
connect(SNSConfig.get_attck_mongo_database(),
host=SNSConfig.get_attck_mongo_host(),
port=SNSConfig.get_attck_mongo_port(),
alias='attck')
except BaseException:
# エラーの場合はデフォルト設定
connect(SNSConfig.DEFAULT_ATTCK_MONGO_DATABASE,
host=SNSConfig.DEFAULT_ATTCK_MONGO_HOST,
port=SNSConfig.DEFAULT_ATTCK_MONGO_PORT,
alias='attck')
def restart_receive_slack_thread():
global th
global rtm_client
el = rtm_client._event_loop
th.end()
th.join()
slack_token = SNSConfig.get_slack_bot_token()
rtm_client = slack.RTMClient(
token=slack_token,
loop=el)
th = SlackThread()
return
def add_like_comment_info(api_user, feed):
# like, comment の情報は リアルタイム更新のため都度取得する
# likes 数を RS から取得
likers = rs.get_likers_from_rs(api_user, feed.package_id)
feed.likes = len(likers)
# like status 取得
mylike = '%s %s' % (SNSConfig.get_sns_identity_name(), api_user)
feed.like = mylike in likers
# comment 数を RS から取得
feed.comments = len(rs.get_comment_from_rs(api_user, feed.package_id))
# feed.save()
return feed
def _make_information_source(self):
# Tool情報作成
tool = ToolInformation()
tool.name = const.SNS_TOOL_NAME
tool.vendor = const.SNS_TOOL_VENDOR
tools = ToolInformationList()
tools.append(tool)
# Identity 作成
identity = Identity(name=SNSConfig.get_sns_identity_name())
# Information Source 作成
information_source = InformationSource()
information_source.tools = tools
information_source.identity = identity
return information_source
def process_message(self, peer, mailfrom, rcptos, data):
# ACCEPT_MAIL_ADDRESS 以外は受け付けない
from ctirs.models import SNSConfig
smtp_accept_mail_address = SNSConfig.get_smtp_accept_mail_address()
if smtp_accept_mail_address not in rcptos:
print('Receiver Mail address is not permitted. ' + str(rcptos))
return
# mailfrom
from django.contrib.auth.models import User
users = User.objects.filter(email__exact=mailfrom)
for user in users:
# user ごとに投稿処理
self.post(user, data)
return
def _is_produced_by_stip_sns_v1(doc):
try:
S_TIP_SNS_IDENTITY_NAME_PREFIX = SNSConfig.get_sns_identity_name()
information_source = doc.stix_header.information_source
identity_name = information_source.identity.name
if not identity_name.startswith(S_TIP_SNS_IDENTITY_NAME_PREFIX):
return False
tool_information = information_source.tools[0]
if tool_information.name != S_TIP_SNS_TOOL_NAME:
return False
if tool_information.vendor != S_TIP_SNS_TOOL_VENDOR:
return False
return True
except BaseException:
return False
def restart_receive_slack_thread():
from boot_sns import StipSnsBoot
th = StipSnsBoot.get_slack_thread()
slack_rtm_client = StipSnsBoot.get_slack_rtm_client()
slack_web_client = StipSnsBoot.get_slack_web_client()
slack_token = SNSConfig.get_slack_bot_token()
if th:
th.end()
th.join()
if not slack_rtm_client:
loop = asyncio.new_event_loop()
slack_rtm_client = slack.RTMClient(token=slack_token, loop=loop)
else:
slack_rtm_client = slack.RTMClient(token=slack_token,
loop=slack_rtm_client._event_loop)
th = SlackThread(slack_rtm_client)
return slack_web_client, slack_rtm_client, th
def is_produced_by_stip_sns(doc):
try:
#S-TIP SNS 作成 STIX に含まれる Identity の Name の値 の prefix を取得する
S_TIP_SNS_IDENTITY_NAME_PREFIX = SNSConfig.get_sns_identity_name()
information_source = doc.stix_header.information_source
identity_name = information_source.identity.name
#identity が 's-tip-sns' で始まること
if identity_name.startswith(S_TIP_SNS_IDENTITY_NAME_PREFIX) != True:
return False
#ToolInformation の name が 'S-TIP' であること
tool_information = information_source.tools[0]
if tool_information.name != S_TIP_SNS_TOOL_NAME:
return False
#ToolInformation の vendor が 'Fujitsu' であること
if tool_information.vendor != S_TIP_SNS_TOOL_VENDOR:
return False
return True
except:
#SNS 特有フィールドがないので S-TIP SNS 作成ではない
return False
def get_feeds_from_package_from_rs(api_user, package_from_rs):
package_id = package_from_rs['package_id']
uploader_id = package_from_rs['uploader']
produced_str = package_from_rs['produced']
version = package_from_rs['version']
try:
feed = Feed.objects.get(package_id=package_id)
if feed.screen_instance is not None:
if feed.screen_instance == SNSConfig.get_sns_identity_name():
feed.screen_name = feed.user.screen_name
feed.screen_affiliation = feed.user.affiliation
feed.ci = feed.user.ci
if feed.user.region is not None:
feed.region_code = feed.user.region.code
except Feed.DoesNotExist:
feed = Feed.create_feeds_record(api_user, package_id, uploader_id,
produced_str, version)
except Exception as e:
import traceback
traceback.print_exc()
raise e
return feed
def create_feeds_record_v2(api_user, package_id, uploader_id, produced_str, version, feed=None):
# RS から取得した STIX から stix_package 取得する
stix_file_path = rs.get_stix_file_path(api_user, package_id)
with open(stix_file_path, 'r', encoding='utf-8') as fp:
bundle = json.load(fp)
# Feed情報を STIX,RS の API から取得する
if not feed:
feed = Feed()
feed.stix_version = version
package_id = bundle['id']
feed.package_id = package_id
feed.filename_pk = package_id
# STIX から表示情報を取得する
bean, stip_sns = Feed.get_stip_from_stix_package_v2(bundle)
if bean.is_sns:
# SNS 産 STIX である
if bean.instance == SNSConfig.get_sns_identity_name():
# 現在稼働中インスタンスと同一
try:
# STIX 定義の username が存在する
feed.user = STIPUser.objects.get(username=bean.user_name)
# 表示はローカル DB から取得する
Feed.set_screen_value_from_local_db(feed, bean)
except BaseException:
# STIX 定義の username が存在しない → N/A アカウント
feed.user = Feed.get_na_account()
# 表示はSTIX File から取得する
Feed.set_screen_value_from_stix(feed, bean)
# すでにユーザーが削除されている
feed.is_valid_user = False
else:
# 現在稼働中インスタンスと異なる
try:
# インスタンス名と同じアカウント
feed.user = STIPUser.objects.get(username=bean.instance)
# 表示はSTIX File から取得する
Feed.set_screen_value_from_stix(feed, bean)
except BaseException:
# インスタンス名と同じアカウントが存在しない → N/A アカウント
feed.user = Feed.get_na_account()
# 表示はSTIX File から取得する
Feed.set_screen_value_from_stix(feed, bean)
else:
# SNS 産 STIX ではない
reports, identities = Feed.get_reports_and_identities(bundle)
is_found = False
if len(identities) > 0:
for identity in identities:
if 'name' in identity:
try:
# インスタンス名と同じアカウント
feed.user = STIPUser.objects.get(username=identity['name'])
is_found = True
# 表示はローカル DB から取得する
Feed.set_screen_value_from_local_db(feed, bean)
break
except STIPUser.DoesNotExist:
pass
if not is_found:
# インスタンス名と同じアカウントが存在しない → N/A アカウント
feed.user = Feed.get_na_account()
# 表示はローカル DB から取得する
Feed.set_screen_value_from_local_db(feed, bean)
feed.screen_instance = bean.instance
# v1 same
feed.date = Feed.get_datetime_from_string(produced_str)
if stip_sns:
# from x-stip-sns
if 'description' in stip_sns:
feed.post = stip_sns['description']
else:
feed.post = ''
if 'name' in stip_sns:
feed.title = stip_sns['name']
else:
feed.title = ''
if const.STIP_STIX2_PROP_POST in stip_sns:
sns_post = stip_sns[const.STIP_STIX2_PROP_POST]
feed.tlp = sns_post['tlp']
sharing_range = sns_post[const.STIP_STIX2_SNS_POST_SHARING_RANGE_KEY]
if sharing_range.startswith('Group:'):
group_name = sharing_range.split(':')[1].strip()
sharing_range_info = Group.objects.get(
en_name=group_name)
elif sharing_range.startswith('People:'):
people_list = []
accounts = sharing_range.split(':')[1].strip()
for account in accounts.split(','):
try:
people = STIPUser.objects.get(
username=account.strip())
people_list.append(people)
except STIPUser.DoesNotExist:
pass
sharing_range_info = people_list
else:
sharing_range_info = None
else:
feed.tlp = None
sharing_range_info = None
else:
# from not x-stip-sns
feed.post = None
for report in reports:
if 'description' in report:
feed.post = report['description']
break
if not feed.post:
feed.post = 'Post, %s' % (feed.package_id)
feed.title = None
for report in reports:
if 'name' in report:
feed.title = report['name']
break
if not feed.title:
feed.title = package_id
feed.tlp = None
sharing_range_info = None
# Attachement Files 情報取得
if stip_sns:
# S-TIP SNS 作成 STIX
if const.STIP_STIX2_PROP_ATTACHMENT_REFS in stip_sns:
# 一度 feed をsave()する
for attach in stip_sns[const.STIP_STIX2_PROP_ATTACHMENT_REFS]:
feed.save()
attach_bundle_id = attach['bundle']
attach_file = Feed.get_attach_file(
api_user,
attach_bundle_id,
'v2')
if attach_file:
feed.files.add(attach_file)
feed.save()
feed.stix_file_path = stix_file_path
if not feed.tlp:
feed.tlp = feed.user.tlp
if isinstance(sharing_range_info, list):
# sharing_range_info の中は STIPUser list
feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_PEOPLE
feed.save()
for stip_user in sharing_range_info:
feed.sharing_people.add(stip_user)
elif isinstance(sharing_range_info, Group):
feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_GROUP
feed.sharing_group = sharing_range_info
else:
feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_ALL
# v1 same
if bean.region_code is not None:
feed.region_code = bean.region_code
else:
if feed.user.region is not None:
feed.region_code = feed.user.region.code
else:
feed.region_code = ''
# v1 same
if bean.ci is not None:
feed.ci = bean.ci
else:
feed.ci = feed.user.ci
# v1 same
if bean.referred_url is not None:
feed.referred_url = bean.referred_url
if bean.stix2_package_id is not None:
feed.stix2_package_id = bean.stix2_package_id
feed.save()
return feed
class CommonExtractor(object):
parentheses_reg_expression = r'\((?P<content>.+)\)'
parentheses_reg = re.compile(parentheses_reg_expression)
square_bracket_reg_expression = r'\[(?P<content>.+)\]'
square_bracket_reg = re.compile(square_bracket_reg_expression)
curly_reg_expression = r'\{(?P<content>.+)\}'
curly_reg = re.compile(curly_reg_expression)
# TLD 判定インスタンス
try:
public_suffix_list_file_path = SNSConfig.get_sns_public_suffix_list_file_path(
)
except BaseException:
# 設定 DB から取得できない場合はデフォルト値
public_suffix_list_file_path = SNSConfig.DEFAULT_SNS_PUBLIC_SUFFIX_LIST_FILE_PATH
tld = TLD(public_suffix_list_file_path)
# object,title から Indicator 作成
@staticmethod
def get_indicator_from_object(object_, title, user_timezone):
# Observableを作成する
observable = Observable()
observable.object_ = object_
# observable,description,titleを設定する
indicator = Indicator()
indicator.timestamp = datetime.datetime.now(tz=user_timezone)
indicator.title = title
indicator.description = title
indicator.observable = observable
return indicator
# cve番号からExploitTarget作成
@staticmethod
def get_exploit_target_from_cve(cve):
title = cve
# description は mitreのページヘのリンク
description = 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=' + str(
cve)
# ExploitTarget
et = ExploitTarget()
et.title = title
et.description = description
et.short_description = description
# Vulnerability
vulnerablity = Vulnerability()
vulnerablity.title = title
vulnerablity.description = description
vulnerablity.short_description = description
vulnerablity.cve_id = cve
et.add_vulnerability(vulnerablity)
return et
# 辞書にすでにそのタイプの値が含まれているかチェックする
# 存在する場合はTrue,しない場合はFalse
# 更新された辞書と一緒にtupleで返却
@staticmethod
def is_duplicate(d, type_, value):
if type_ not in d:
# 辞書にそのタイプが初出の場合はリストを作成
d[type_] = [value]
return False, d
else:
# 辞書にそのタイプが存在する場合は重複している
if value in d[type_]:
return True, d
else:
d[type_].append(value)
return False, d
@staticmethod
def is_punctuation_char(c):
if ((c == '.') or (c == ',') or (c == '!') or (c == '?') or (c == ';')
or (c == ':')):
return True
return False
@staticmethod
def remove_punctuation_char(word):
def remove_bracket(word, reg):
ret = reg.match(word)
if ret is not None:
return ret.group('content')
else:
return word
# puctuation 対応
if (len(word) == 0):
return ''
if (len(word) == 1):
if CommonExtractor.is_punctuation_char(word):
return ''
last_char = word[-1]
if CommonExtractor.is_punctuation_char(last_char):
word = word[:-1]
# () を外す
word = remove_bracket(word, CommonExtractor.parentheses_reg)
# [] を外す
word = remove_bracket(word, CommonExtractor.square_bracket_reg)
# {} を外す
word = remove_bracket(word, CommonExtractor.curly_reg)
return word
# 単語に cve 情報が含まれていたら返却する
@staticmethod
def get_cve_from_word(word):
# puctuation 対応
word = CommonExtractor.remove_punctuation_char(word)
if len(word) == 0:
return None
# cveか?
v = CommonExtractor._get_cve_value(word)
if v is not None:
return v
return None
# 単語に actor 情報が含まれていたら返却する
@staticmethod
def get_ta_from_words(words, actors_list=[]):
# actorか?
for actor_words in actors_list:
# actors_list 一つひとつの項目(actor_words)ごとに以下のチェック
# actor_words は空白区切りでの複数ワードの可能性がある
actor_word_list = actor_words.split(' ')
index = 0
isMatch = True
# actor_words の単語一つづつごとに words をずらしてチェックする
for actor_word in actor_word_list:
try:
# puctuation 対応
while True:
word = CommonExtractor.remove_punctuation_char(
words[index])
# puctuation 対応後に長さが 0 の場合, words[index] は puctuation 文字であるため index をすすめる
if len(word) == 0:
index += 1
else:
break
# 大文字小文字は区別しない
if actor_word.lower() != word.lower():
# 単語が違う (次の actor_words checkを行う)
isMatch = False
break
except IndexError:
# word が終端を迎えたので違う (次の actor_words checkを行う)
isMatch = False
break
# 単語が一致したのでずらす
index += 1
# 最終的に一致したら actors_wordsを返却
if isMatch:
return actor_words
# 一致しなかった
return None
# 単語がそれぞれipv4,url,hash,domainであるかを判定する
# そのcybox 種別 と 値を返却する
@staticmethod
def get_object_from_word(word):
# word の両端 check
if (word[0] == '\"' and word[-1] == '\"') or (word[0] == '\''
and word[-1] == '\''):
# 両端に " か ' の場合だけ削除
word = word[1:-1]
# puctuation 対応
word = CommonExtractor.remove_punctuation_char(word)
if len(word) == 0:
return None, None
# ipv4か?
v = CommonExtractor._get_ipv4_value(word)
if v is not None:
return JSON_OBJECT_TYPE_IPV4, v.replace('[', '').replace(']', '')
# urlか?
v = CommonExtractor._get_url_value(word)
if v is not None:
return JSON_OBJECT_TYPE_URI, v
# hash 値は長い順番から判定する
# sha256か?
v = CommonExtractor._get_sha256_value(word)
if v is not None:
return JSON_OBJECT_TYPE_SHA256, v
# sha1か?
v = CommonExtractor._get_sha1_value(word)
if v is not None:
return JSON_OBJECT_TYPE_SHA1, v
# md5か?
v = CommonExtractor._get_md5_value(word)
if v is not None:
return JSON_OBJECT_TYPE_MD5, v
# email_addressか?
v = CommonExtractor._get_email_address_value(word)
if v is not None:
return JSON_OBJECT_TYPE_EMAIL_ADDRESS, v
# domainか?
v = CommonExtractor._get_domain_value(word)
if v is not None:
if CommonExtractor.is_file_name(v):
return JSON_OBJECT_TYPE_FILE_NAME, v
else:
# TLD が含まれていたらドメイン名と判断
v = v.replace('[', '').replace(']', '')
if CommonExtractor.tld.get_tld(v) is not None:
# ドメイン名とする
return JSON_OBJECT_TYPE_DOMAIN, v
else:
# ファイル名とする
return JSON_OBJECT_TYPE_FILE_NAME, v
# file_nameか?
v = file_name_reg.match(word)
if v is not None:
# 最初に見つかった項目のみを対象とする
return JSON_OBJECT_TYPE_FILE_NAME, v.group(1)
return None, None
# web 画面から取得した indicators json から stix indicators 作成する
@staticmethod
def get_indicator_from_json(indicator_json, user_timezone):
type_ = indicator_json['type']
v = indicator_json['value']
title = indicator_json['title']
o_ = None
# ipv4か?
if type_ == JSON_OBJECT_TYPE_IPV4:
o_ = Address()
o_.address_value = v.replace('[', '').replace(']', '')
# urlか?
if type_ == JSON_OBJECT_TYPE_URI:
o_ = URI()
o_.value = v
# md5か?
if type_ == JSON_OBJECT_TYPE_MD5:
o_ = File()
o_.md5 = v
# sha1か?
if type_ == JSON_OBJECT_TYPE_SHA1:
o_ = File()
o_.sha1 = v
# sha256か?
if type_ == JSON_OBJECT_TYPE_SHA256:
o_ = File()
o_.sha256 = v
# sha512か?
if type_ == JSON_OBJECT_TYPE_SHA512:
o_ = File()
o_.sha512 = v
# email-addressか?
if type_ == JSON_OBJECT_TYPE_EMAIL_ADDRESS:
o_ = EmailAddress()
o_.address_value = v
# domainか?
if type_ == JSON_OBJECT_TYPE_DOMAIN:
o_ = DomainName()
o_.value = v.replace('[', '').replace(']', '')
# file名か?
if type_ == JSON_OBJECT_TYPE_FILE_NAME:
o_ = File()
o_.file_name = v
# なにも該当していないので None
if o_ is None:
print('何も該当なし:' + str(type_) + ':' + str(v))
return None
# indicator 作って返却
indicator_title = '%s (%s)' % (v, title)
ind = CommonExtractor.get_indicator_from_object(
o_, indicator_title, user_timezone)
return ind
# web 画面から取得した ttp json から stix ttp 作成する
@staticmethod
def get_exploit_target_from_json(ttp_json):
json_cve = ttp_json['value']
json_title = ttp_json['title']
# title は "%CVE番号% (index)" とする
title = '%s (%s)' % (json_cve, json_title)
# CVE 情報を circl から取得する
cve_info = Cve.get_cve_info(json_cve)
# 各種 CVE 情報のリンクを作成
mitre_url = 'https://cve.mitre.org/cgi-bin/cvename.cgi?name=' + str(
json_cve)
circl_url = 'http://cve.circl.lu/cve/' + str(json_cve)
# Expoit_Target, Vulnerability の Short Description は link
common_short_description = '%s (<a href="%s" target="_blank">MITRE</a>, <a href="%s" target="_blank">circl.lu</a>)<br/>' % (
json_cve, mitre_url, circl_url)
# base_score
try:
vul_cvss_score = CVSSVector()
vul_cvss_score.base_score = cve_info['cvss']
except BaseException:
vul_cvss_score = None
# Expoit_Target, Vulnerability の Description 作成
common_decritpion = common_short_description
# base_score があったら追加する
if vul_cvss_score is not None:
common_decritpion += ('Base Score: %s<br/>' %
(vul_cvss_score.base_score))
# vulnerability の description は circl から取得した description
try:
common_decritpion += ('%s<br/>' % (cve_info['summary']))
except BaseException:
# 取得失敗時は circl のページの url
common_decritpion += ('%s<br/>' % (circl_url))
# ExploitTarget
et = ExploitTarget()
et.title = title
et.description = common_decritpion
et.short_description = common_short_description
# Vulnerability
vulnerablity = Vulnerability()
vulnerablity.title = title
vulnerablity.description = common_decritpion
vulnerablity.short_description = common_short_description
vulnerablity.cve_id = json_cve
if vul_cvss_score is not None:
vulnerablity.cvss_score = vul_cvss_score
et.add_vulnerability(vulnerablity)
return et
# value , descirption から ThreatActorObject 作成する
@staticmethod
def _get_threat_actor_object(value,
description=None,
crowd_strike_motivations=[]):
# 攻撃者情報作成
organisation_name = OrganisationName(value)
party_name = PartyName()
party_name.add_organisation_name(organisation_name)
identity_specification = STIXCIQIdentity3_0()
identity_specification.party_name = party_name
identity = CIQIdentity3_0Instance()
# ThreatActor
ta = ThreatActor()
ta.identity = identity
ta.identity.specification = identity_specification
# Title に抽出した Threat Actor 名前
ta.title = value
ta.description = description
ta.short_description = description
ta.identity = identity
# motivations 作成
for crowd_strike_motivation in crowd_strike_motivations:
ta_motivation = Statement(crowd_strike_motivation['value'])
# motivation 追加
ta.add_motivation(ta_motivation)
return ta
# regをもとにitemを解析し、最初に見つかった文字列を返却
# 存在しない場合はNone
@staticmethod
def _get_regular_value(reg, item):
v = reg.match(item)
if v is not None:
# 最初に見つかった項目のみを対象とする
r = v.group(1)
return r
return None
# 文字列がipv4を含む場合、最初に見つかった文字列を返却
# 存在しない場合はNone
@staticmethod
def _get_ipv4_value(item):
v = ipv4_reg.match(item)
if v is not None:
return v.group(1)
return None
# 文字列がurlを含む場合、: の前後に [] がある場合は取り除いて返却
# 存在しない場合はNone
@staticmethod
def _get_url_value(item):
v = url_reg.match(item)
if v is not None:
return v.group(1) + v.group(2) + v.group(3)
return None
# 文字列がmd5を含む場合、最初に見つかった文字列を返却
# 存在しない場合はNone
@staticmethod
def _get_md5_value(item):
return CommonExtractor._get_regular_value(md5_reg, item)
# 文字列がsha1を含む場合、最初に見つかった文字列を返却
# 存在しない場合はNone
@staticmethod
def _get_sha1_value(item):
return CommonExtractor._get_regular_value(sha1_reg, item)
# 文字列がsha256を含む場合、最初に見つかった文字列を返却
# 存在しない場合はNone
@staticmethod
def _get_sha256_value(item):
return CommonExtractor._get_regular_value(sha256_reg, item)
# 文字列がsha512を含む場合、最初に見つかった文字列を返却
# 存在しない場合はNone
@staticmethod
def _get_sha512_value(item):
return CommonExtractor._get_regular_value(sha512_reg, item)
# 文字列がemail-addressを含む場合、最初に見つかった文字列を返却
# 存在しない場合はNone
@staticmethod
def _get_email_address_value(item):
return CommonExtractor._get_regular_value(email_address_reg, item)
# 文字列がdomainを含む場合、最初に見つかった文字列を返却
# 存在しない場合はNone
@staticmethod
def _get_domain_value(item):
return CommonExtractor._get_regular_value(domain_reg, item)
# domain候補の文字列がファイル名であるかをチェックする
@staticmethod
def is_file_name(s):
tld = s.split('.')[-1]
return tld.lower() in file_name_extentions
# 文字列がcveを含む場合、最初に見つかった文字列を返却
# 存在しない場合はNone
@staticmethod
def _get_cve_value(item):
return CommonExtractor._get_regular_value(cve_reg, item)
def get_white_list(request):
return get_merged_conf_list(SNSConfig.get_common_white_list(),
request.user.sns_profile.indicator_white_list)
def get_threat_actors_list(request):
return get_merged_conf_list(SNSConfig.get_common_ta_list(),
request.user.sns_profile.threat_actors)
def call_jira(request):
try:
# JIRA が import されていない場合は何もしない
if imported_jira is None:
rsp = {}
return JsonResponse(rsp)
# feed情報取得
feed_file_name_id = request.GET['feed_id']
package_id_arg = request.GET['package_id']
feed = Feed.get_feeds_from_package_id(request.user, package_id_arg)
# JIRA instance
proxies = System.get_request_proxies()
j = JIRA(server=SNSConfig.get_jira_host(),
proxies=proxies,
basic_auth=(SNSConfig.get_jira_username(),
SNSConfig.get_jira_password()))
# issues作成
issue = j.create_issue(project=SNSConfig.get_jira_project(),
summary=feed.title,
description=feed.post,
issuetype={'name': SNSConfig.get_jira_type()})
# 添付があればそれもつける
for attach_file in feed.files.all():
file_path = Feed.get_attach_file_path(attach_file.package_id)
j.add_attachment(issue=issue,
attachment=file_path,
filename=str(attach_file.file_name))
# STIX添付
stix_package = STIXPackage.from_xml(feed.stix_file_path)
package_id = stix_package.id_
stix_file_name = '%s.xml' % (package_id)
j.add_attachment(issue=issue,
attachment=feed.stix_file_path,
filename=stix_file_name)
# CSV添付
# CSVの中身を取得する
content = get_csv_content(feed_file_name_id)
csv_attachment = io.StringIO()
csv_attachment.write(content)
csv_file_name = '%s.csv' % (package_id)
j.add_attachment(issue=issue,
attachment=csv_attachment,
filename=csv_file_name)
# PDF添付
feed_pdf = FeedPDF(feed, stix_package)
pdf_attachment = io.BytesIO()
feed_pdf.make_pdf_content(pdf_attachment, feed)
pdf_file_name = '%s.pdf' % (package_id)
j.add_attachment(issue=issue,
attachment=pdf_attachment,
filename=pdf_file_name)
# isssue番号返却
url = SNSConfig.get_jira_host(
) + '/projects/' + SNSConfig.get_jira_project() + '/issues/' + str(
issue)
rsp = {
'issues': str(issue),
'url': url,
}
return JsonResponse(rsp)
except Exception as e:
traceback.print_exc()
return HttpResponseServerError(str(e))
def create_feeds_record(api_user, package_id, uploader_id, produced_str):
# RS から取得した STIX から stix_package 取得する
stix_file_path = rs.get_stix_file_path(api_user, package_id)
stix_package = STIXPackage.from_xml(stix_file_path, encoding='utf-8')
# Feed情報を STIX,RS の API から取得する
feed = Feed()
feed.package_id = package_id
feed.filename_pk = rs.convert_package_id_to_filename(package_id)
# STIX から表示情報を取得する
bean = Feed.get_stip_from_stix_package(stix_package)
if bean.is_sns:
# SNS 産 STIX である
if bean.instance == SNSConfig.get_sns_identity_name():
# 現在稼働中インスタンスと同一
try:
# STIX 定義の username が存在する
feed.user = STIPUser.objects.get(username=bean.user_name)
# 表示はローカル DB から取得する
Feed.set_screen_value_from_local_db(feed, bean)
except BaseException:
# STIX 定義の username が存在しない → N/A アカウント
feed.user = Feed.get_na_account()
# 表示はSTIX File から取得する
Feed.set_screen_value_from_stix(feed, bean)
# すでにユーザーが削除されている
feed.is_valid_user = False
else:
# 現在稼働中インスタンスと異なる
try:
# インスタンス名と同じアカウント
feed.user = STIPUser.objects.get(username=bean.instance)
# 表示はSTIX File から取得する
Feed.set_screen_value_from_stix(feed, bean)
except BaseException:
# インスタンス名と同じアカウントが存在しない → N/A アカウント
feed.user = Feed.get_na_account()
# 表示はSTIX File から取得する
Feed.set_screen_value_from_stix(feed, bean)
else:
# SNS 産 STIX ではない
if bean.instance is not None:
# instance がある
instance_user_name = bean.instance.replace(' ', '')
try:
# インスタンス名と同じアカウント
feed.user = STIPUser.objects.get(username=instance_user_name)
# 表示はローカル DB から取得する
Feed.set_screen_value_from_local_db(feed, bean)
except BaseException:
# インスタンス名と同じアカウントが存在しない → N/A アカウント
feed.user = Feed.get_na_account()
# 表示は bean.instance [bean.instance]
feed.screen_name = bean.instance
feed.screen_instance = bean.instance
else:
# instance がない
# N/A アカウント
feed.user = Feed.get_na_account()
# 表示はローカル DB(N/Aアカウント) から取得する
Feed.set_screen_value_from_local_db(feed, bean)
feed.date = Feed.get_datetime_from_string(produced_str)
feed.post = stix_package.stix_header.description
if feed.post is None:
feed.post = ''
# Attachement Files 情報取得
if Feed.is_stip_sns_stix_package(stix_package):
# S-TIP SNS 作成 STIX
if stix_package.related_packages is not None:
# 一度 feed をsave()する
feed.save()
# Related_packages は SNS STIX 以外の可能性もある
for related_package in stix_package.related_packages:
# attachement は attachdirにいれるべきその時のファイル名は attachment_stix_idであるべき
attach_file = Feed.get_attach_file(api_user, related_package.item.id_)
# attach_file が None の場合は Attach File ではない
if attach_file is None:
continue
feed.files.add(attach_file)
feed.save()
feed.title = stix_package.stix_header.title
feed.stix_file_path = stix_file_path
try:
uploader_stipuser = STIPUser.objects.get(id=uploader_id)
feed.tlp = Feed.get_ais_tlp_from_stix_header(stix_package.stix_header, uploader_stipuser.tlp)
if feed.tlp is None:
# 取得ができなかった場合は default TLP の AMBER
feed.tlp = 'AMBER'
except BaseException:
# uploader_profile が存在しない場合は default TLP の AMBER
feed.tlp = 'AMBER'
sharing_range_info = Feed.get_sharing_range_from_stix_header(stix_package.stix_header)
if isinstance(sharing_range_info, list):
# sharing_range_info の中は STIPUser list
feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_PEOPLE
feed.save()
for stip_user in sharing_range_info:
feed.sharing_people.add(stip_user)
elif isinstance(sharing_range_info, Group):
feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_GROUP
feed.sharing_group = sharing_range_info
else:
feed.sharing_range_type = const.SHARING_RANGE_TYPE_KEY_ALL
# feed.package_id = package_id
if bean.region_code is not None:
feed.region_code = bean.region_code
else:
if feed.user.region is not None:
feed.region_code = feed.user.region.code
else:
feed.region_code = ''
if bean.ci is not None:
feed.ci = bean.ci
else:
feed.ci = feed.user.ci
if bean.referred_url is not None:
feed.referred_url = bean.referred_url
if bean.stix2_package_id is not None:
feed.stix2_package_id = bean.stix2_package_id
feed.save()
return feed
