def delete_stix_document(id_=None, package_id=None):
if id_:
origin_path = StixFiles.delete_by_id(id_)
elif package_id:
origin_path = StixFiles.delete_by_package_id(package_id)
else:
return
# ファイル削除
if os.path.exists(origin_path):
os.remove(origin_path)
return
def regist(stix_file_path, community, via, package_name=None):
package_bean = get_package_bean(stix_file_path)
if package_name:
package_bean.package_name = package_name
try:
with open(stix_file_path, 'rb') as fp:
stix_file_doc = StixFiles.create(package_bean, community, fp, via)
except BaseException:
os.remove(stix_file_path)
traceback.print_exc()
raise Exception('duplicate package_id:%s' % (package_bean.package_id))
try:
path = os.path.join(community.dir_path, COMMUNITY_ORIGIN_DIR_NAME,
str(stix_file_doc.content.grid_id))
shutil.move(stix_file_path, path)
stix_file_doc.origin_path = path
stix_file_doc.save()
except Exception as e:
stix_file_doc.delete()
traceback.print_exc()
raise Exception(str(e))
stix_file_doc.split_child_nodes()
webhook_create(community, stix_file_doc)
for cl in Communities.get_clients_from_community(community):
cl.push(stix_file_doc)
return
def rebuild_cache(request):
if request.method != 'GET':
return error_page_free_format(request, 'invalid method')
# activeユーザー以外はエラー
if not request.user.is_active:
return error_page_inactive(request)
# is_admin権限なしの場合はエラー
if not request.user.is_admin:
return error_page_no_view_permission(request)
try:
# レンダリング
StixFiles.rebuild_cache()
replace_dict = get_success_replace_dict(request)
replace_dict['info_msg'] = 'Rebuild Success!!'
return render(request, 'system.html', replace_dict)
except Exception:
# エラーページ
return error_page(request)
def delete_stix_files_id(request, package_id):
try:
origin_path = StixFiles.delete_by_package_id(package_id)
if os.path.exists(origin_path):
os.remove(origin_path)
return JsonResponse({}, status=204)
except Exception as e:
traceback.print_exc()
return error(e)
def delete_record_related_packages(package_id=None):
try:
feeds_ = Feed.objects.filter(package_id=package_id)
attach_package_ids = []
if feeds_:
for feed_ in feeds_:
for file_ in feed_.files.all():
attach_package_ids.append(file_.package_id)
from ctirs.core.mongo.documents_stix import StixFiles
StixFiles.delete_by_package_id(file_.package_id)
file_.delete()
for attach_package_id in attach_package_ids:
attach_dir = Feed.get_attach_stix_dir_path(attach_package_id)
if os.path.isdir(attach_dir):
shutil.rmtree(attach_dir)
Feed.objects.filter(package_id=package_id).delete()
return
except Exception as e:
print(e)
def get_read_collection_content():
debug_print('>>>enter get_read_collection_content')
j = []
for stix_file in StixFiles.objects(input_community=_community):
debug_print('>>>stix_file id: ' + str(stix_file.id))
content = stix_file.content.read()
j.append(json.loads(content))
return JsonResponse(j,
safe=False,
content_type=RESPONSE_COMMON_CONTENT_TYPE_STIX_JSON)
def get_package_bean_v2(stix_file_path):
try:
with codecs.open(stix_file_path, 'r', encoding='utf-8') as fp:
content = fp.read()
doc = json.loads(content)
package_bean = StixFiles.PackageBean()
package_bean.package_id = doc['id']
if ('spec_version' in doc):
package_bean.version = doc['spec_version']
else:
package_bean.version = '2.1'
stip_sns = _is_produced_by_stip_sns_v2(doc)
package_bean.related_packages = None
produced_str = None
if stip_sns:
package_bean.package_name = stip_sns['name']
package_bean.description = stip_sns['description']
package_bean.is_created_by_sns = True
package_bean.sns_type = _get_stip_sns_type_v2(stip_sns)
if package_bean.sns_type != StixFiles.STIP_SNS_TYPE_V2_POST:
package_bean.is_post_sns = False
package_bean.related_packages = []
if const.STIP_STIX2_PROP_OBJECT_REF in stip_sns:
package_bean.related_packages.append(stip_sns[const.STIP_STIX2_PROP_OBJECT_REF])
if const.STIP_STIX2_PROP_ATTACHMENT_REFS in stip_sns:
for ref in stip_sns[const.STIP_STIX2_PROP_ATTACHMENT_REFS]:
package_bean.related_packages.append(ref['bundle'])
if len(package_bean.related_packages) == 0:
package_bean.related_packages = None
produced_str = stip_sns['created']
else:
package_bean.package_name = None
package_bean.description = None
report = _get_report_object(doc)
if report:
package_bean.package_name = report['name']
if 'description' in report:
package_bean.description = report['description']
produced_str = report['created']
if not package_bean.package_name:
package_bean.package_name = package_bean.package_id
if not package_bean.description:
package_bean.description = 'Post: %s' % (package_bean.package_id)
package_bean.is_created_by_sns = False
package_bean.is_post_sns = True
_set_stix_bean_from_doc_v2(package_bean, doc)
if produced_str:
package_bean.produced = stix2_str_to_datetime(produced_str)
else:
package_bean.produced = datetime.datetime.now(tz=pytz.utc)
return package_bean
except Exception as e:
raise Exception('Can\'t parse STIX. ' + e.message)
def likers(request):
try:
#GET 以外はエラー
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
#引数取得
original_package_id = get_comments_original_package_id(request)
if original_package_id is None:
print '/api/v1/sns/likers package_id is None.'
return HttpResponseNotFound()
liker_dict = {}
#original_package_id を related_packages リスト要素に含む stix_fileを返却
for stix_file in StixFiles.objects(
Q(related_packages=original_package_id)
& Q(sns_type=StixFiles.STIP_SNS_TYPE_LIKE)):
liker = stix_file.get_unique_name()
if liker is not None:
if liker_dict.has_key(liker):
liker_dict[liker] += 1
else:
liker_dict[liker] = 1
#original_package_id を related_packages リスト要素に含む stix_fileを返却
for stix_file in StixFiles.objects(
Q(related_packages=original_package_id)
& Q(sns_type=StixFiles.STIP_SNS_TYPE_UNLIKE)):
unliker = stix_file.get_unique_name()
if unliker is not None:
if liker_dict.has_key(unliker):
liker_dict[unliker] -= 1
else:
liker_dict[unliker] = -1
likers = []
for liker, value in liker_dict.iteritems():
if value == 1:
likers.append(liker)
return JsonResponse(likers, safe=False)
except Exception as e:
import traceback
traceback.print_exc()
return error(e)
def query(request):
try:
#GET 以外はエラー
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
#引数取得
query_string = get_query_query_string(request)
#query_string 未指定時は空リスト返却
if query_string is None:
r = {}
r['feeds'] = []
return JsonResponse(r, safe=False)
#返却条件は SNS に返却 かつ version 2.0 以外
QQ = Q(is_post_sns__ne=False) & Q(version__ne='2.0')
#package_name か post に query_string が含まれているか (大文字小文字区別せず)
QQ &= (Q(package_name__icontains=query_string)
| Q(post__icontains=query_string))
stix_files = set([])
#Query
for stix_file in StixFiles.objects(QQ).order_by('-produced').timeout(
False):
stix_files.add(stix_file)
#ObservableCache から query_string を含む StixFile の set を作成
for observable_cache in ObservableCaches.objects.filter(
Q(value__icontains=query_string)):
stix_file = StixFiles.objects.get(
package_id=observable_cache.package_id)
stix_files.add(stix_file)
#時間でソートする
stix_files = sorted(list(stix_files),
key=lambda s: s.produced,
reverse=True)
#返却データ作成
l = []
for stix_file in stix_files:
l.append(
get_return_dictionary_from_stix_file_document(stix_file,
content=content))
r = {}
r['feeds'] = l
return JsonResponse(r, safe=False)
except Exception as e:
import traceback
traceback.print_exc()
return error(e)
def delete_stix_related_document(package_id=None):
if package_id:
# mongoのdocument削除
origin_paths, remove_package_ids = StixFiles.delete_by_related_packages(
package_id)
# ファイル削除
for origin_path in origin_paths:
try:
os.remove(origin_path)
# ファイルが見つからない、ディレクトリのときは無視する
except FileNotFoundError:
pass
except IsADirectoryError:
pass
return remove_package_ids
def delete_stix_files_id(request,package_id):
try:
#認証する
user = authentication(request)
if user is None:
return error(Exception('You have no permission for this operation.'))
#mongoから該当レコード削除
origin_path = StixFiles.delete_by_package_id(package_id)
#ファイル削除
if os.path.exists(origin_path) == True:
os.remove(origin_path)
#response data 作成
return JsonResponse({},status=204)
except Exception as e:
traceback.print_exc()
return error(e)
def get_stix_files_from_mongo(self, collection_name, start_time, end_time):
from ctirs.core.mongo.documents_stix import StixFiles
# TAXII Server 設定取得
ts = TaxiiServers.objects.get(collection_name=collection_name)
# 条件絞り込み
QQ = Q(input_community__in=ts.communities)
QQ = QQ & Q(version__startswith='1.')
if start_time is not None and end_time is not None:
QQ = QQ & Q(produced__gt=start_time)
QQ = QQ & Q(produced__lt=end_time)
elif start_time is None and end_time is not None:
QQ = QQ & Q(produced__lt=end_time)
elif start_time is not None and end_time is None:
QQ = QQ & Q(produced__gt=start_time)
return StixFiles.objects(QQ)
def regist(stix_file_path, community, via, package_name=None):
#stixの中身から STIX の情報を取得
package_bean = get_package_bean(stix_file_path)
#登録するパッケージ名はweb form指定があれば優先的に使用する
if package_name is not None:
package_bean.package_name = package_name
#STIXFileコレクションに保存
try:
#StixFileドキュメントを作成
with open(stix_file_path, 'r') as fp:
stix_file_doc = StixFiles.create(package_bean, community, fp, via)
except: #おもにduplicateエラー処理 #stixは削除
os.remove(stix_file_path)
traceback.print_exc()
raise Exception('duplicate package_id:%s' % (package_bean.package_id))
#Fileをコミュニティ別のディレクトリに移動
try:
#file名はGridFsの方のidとする
#stixファイルを移動
path = os.path.join(community.dir_path, COMMUNITY_ORIGIN_DIR_NAME,
str(stix_file_doc.content.grid_id))
shutil.move(stix_file_path, path)
#path保存
stix_file_doc.origin_path = path
stix_file_doc.save()
except Exception as e:
#ドキュメント作成後のエラーはドキュメントを削除する
#grid_fsもcascade削除される動作は確認済
stix_file_doc.delete()
traceback.print_exc()
raise Exception(str(e))
#Node分割+キャッシュ作成
stix_file_doc.split_child_nodes()
#CommunityのWebHooksの数だけPOSTする
webhook_create(community, stix_file_doc)
#push
from ctirs.core.mongo.documents import Communities
for cl in Communities.get_clients_from_community(community):
cl.push(stix_file_doc)
return
def related_packages(request):
try:
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
original_package_id = get_comments_original_package_id(request)
if original_package_id is None:
return HttpResponseNotFound()
related_packages_list = []
for stix_file in StixFiles.objects(
related_packages=original_package_id):
related_packages_list.append(
get_return_dictionary_from_stix_file_document(stix_file))
return JsonResponse(related_packages_list, safe=False)
except Exception as e:
import traceback
traceback.print_exc()
return error(e)
def delete(request):
if request.method != 'GET':
return error_page_free_format(request, 'Invalid HTTP Method.')
#activeユーザー以外はエラー
if request.user.is_active == False:
return error_page_inactive(request)
#削除対象 ID が ,区切り文字列で渡る
ids = get_list_delete_id(request).split(',')
try:
for id_ in ids:
#mongoから該当レコード削除
origin_path = StixFiles.delete_by_id(id_)
#ファイル削除
if os.path.exists(origin_path) == True:
os.remove(origin_path)
return top(request)
except Exception:
return error_page(request)
return top(request)
def related_packages(request):
try:
#GET 以外はエラー
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
#引数取得
original_package_id = get_comments_original_package_id(request)
if original_package_id is None:
print '/api/v1/sns/related_packages package_id is None.'
return HttpResponseNotFound()
l = []
#original_package_id を related_packages リスト要素に含む stix_fileを返却
for stix_file in StixFiles.objects(
related_packages=original_package_id):
l.append(get_return_dictionary_from_stix_file_document(stix_file))
return JsonResponse(l, safe=False)
except Exception as e:
import traceback
traceback.print_exc()
return error(e)
def delete(request):
if request.method != 'GET':
return error_page_free_format(request, 'Invalid HTTP Method.')
# activeユーザー以外はエラー
if not request.user.is_active:
return error_page_inactive(request)
# 削除対象 ID が ,区切り文字列で渡る
ids = get_list_delete_id(request).split(',')
# is_admin権限なしの場合はエラー
if not request.user.is_admin:
return error_page_no_view_permission(request)
try:
for id_ in ids:
# mongoから該当レコード削除
origin_path = StixFiles.delete_by_id(id_)
# ファイル削除
if os.path.exists(origin_path):
os.remove(origin_path)
return redirect('list')
except Exception:
return error_page(request)
return redirect('list')
def comments(request):
try:
# GET 以外はエラー
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
# 引数取得
original_package_id = get_comments_original_package_id(request)
if original_package_id is None:
print('/api/v1/sns/comments package_id is None.')
return HttpResponseNotFound()
comments_list = []
# original_package_id を related_packages リスト要素に含み sns_type が comment の stix_fileを返却
for stix_file in StixFiles.objects(
Q(related_packages=original_package_id)
& Q(sns_type=StixFiles.STIP_SNS_TYPE_COMMENT)):
comments_list.append(
get_return_dictionary_from_stix_file_document(stix_file))
return JsonResponse(comments_list, safe=False)
except Exception as e:
import traceback
traceback.print_exc()
return error(e)
def get_stix_files_from_mongo(self, collection_name, start_time, end_time):
# print '>>>get_stix_files_from_mongo:start_time:' + str(start_time)
# print '>>>get_stix_files_from_mongo:end_time:' + str(end_time)
# print '>>>get_stix_files_from_mongo:collection_name:' + str(collection_name)
# TAXII Server 設定取得
ts = TaxiiServers.objects.get(collection_name=collection_name)
# 条件絞り込み
QQ = Q(information_source__in=ts.information_sources)
# for is_ in ts.information_sources:
# print '>>>get_stix_files_from_mongo:information_sources:' + str(is_.name)
# print '>>>get_stix_files_from_mongo:information_sources:' + str(is_.id)
if start_time is not None and end_time is not None:
QQ = QQ & Q(produced__gt=start_time)
QQ = QQ & Q(produced__lt=end_time)
elif start_time is None and end_time is not None:
QQ = QQ & Q(produced__lt=end_time)
elif start_time is not None and end_time is None:
QQ = QQ & Q(produced__gt=start_time)
# print '>>>get_stix_files_from_mongo:count:' + str(StixFiles.objects(QQ).count())
return StixFiles.objects(QQ)
def get_package_bean_v1(stix_file_path):
doc = STIXPackage.from_xml(stix_file_path)
try:
package_bean = StixFiles.PackageBean()
package_bean.is_post_sns = True
package_bean.is_created_by_sns = False
sns_type = None
if _is_produced_by_stip_sns_v1(doc):
package_bean.is_created_by_sns = True
sns_type = _get_stip_sns_type_v1(doc)
if sns_type != StixFiles.STIP_SNS_TYPE_ORIGIN:
package_bean.is_post_sns = False
try:
package_bean.related_packages = []
for related_package in doc.related_packages:
package_bean.related_packages.append(related_package.item.id_)
except TypeError:
package_bean.related_packages = None
package_bean.package_id = doc.id_
package_bean.version = doc._version
package_bean.produced = _get_produced_time_stix_1_x(doc)
package_bean.package_name = doc.stix_header.title
package_bean.sns_type = sns_type
try:
package_bean.description = doc.stix_header.description.value
if package_bean.description is None:
package_bean.description = ''
except BaseException:
package_bean.description = ''
_set_stix_bean_from_doc_v1(package_bean, doc)
if package_bean.sns_user_name == '':
package_bean.sns_user_name = _get_sns_user_name_from_instance(
package_bean.sns_instance)
return package_bean
except Exception:
pass
def get_package_bean(stix_file_path):
package_bean = StixFiles.PackageBean()
#STIX 1.1 parse
try:
doc = STIXPackage.from_xml(stix_file_path)
package_bean.is_post_sns = True
package_bean.is_created_by_sns = False
sns_type = None
#S-TIP SNS で作成された STIX であるか?
if is_produced_by_stip_sns(doc) == True:
#SNS 産である
package_bean.is_created_by_sns = True
sns_type = get_stip_sns_type(doc)
#origin 投稿以外は表示しない
if sns_type != StixFiles.STIP_SNS_TYPE_ORIGIN:
package_bean.is_post_sns = False
#realted_packages探す
try:
package_bean.related_packages = []
for related_package in doc.related_packages:
package_bean.related_packages.append(related_package.item.id_)
except TypeError:
package_bean.related_packages = None
package_bean.package_id = doc.id_
package_bean.version = doc._version
package_bean.produced = get_produced_time_stix_1_x(doc)
package_bean.package_name = doc.stix_header.title
package_bean.sns_type = sns_type
try:
package_bean.description = doc.stix_header.description.value
if package_bean.description is None:
package_bean.description = ''
except:
package_bean.description = ''
#S-TIP SNS 作成の STIX から pacakge_bean の値をセットする
set_stix_bean_from_doc(package_bean, doc)
#SNS 産以外は sns_user_name が設定されていないので instance 名から取得する
if package_bean.sns_user_name == '':
package_bean.sns_user_name = get_sns_user_name_from_instance(
package_bean.sns_instance)
return package_bean
except Exception:
pass
#STIX 2.0 parse
try:
with codecs.open(stix_file_path, 'r', 'utf-8') as fp:
content = fp.read()
doc = json.loads(content)
package_bean.package_name = None
#最初に見つかったtypeがreportのnameをpackage_nameとする
#また、STIX 2.0 では package の timestampの格納場所がないのでNoneとする
produced_str = None
for d in doc['objects']:
if d['type'] == 'report':
package_bean.package_name = d['name']
produced_str = d['created']
package_bean.package_id = doc['id']
if doc.has_key('spec_version') == True:
package_bean.version = doc['spec_version']
else:
#STIX 2.1 には spec_version がない
package_bean.version = '2.1'
#Produced Time は Report の produced time
if produced_str is not None:
package_bean.produced = stix2_str_to_datetime(produced_str)
else:
package_bean.produced = datetime.datetime.now()
package_bean.is_post_sns = True
package_bean.is_created_by_sns = False
package_bean.related_packages = None
return package_bean
except Exception as e:
traceback.print_exc()
raise Exception('Can\'t parse STIX. ' + e.message)
def feeds(request):
try:
#GET 以外はエラー
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
#引数取得
start_time = get_feeds_start_time(request)
last_feed = get_feeds_last_feed(request)
range_big_datetime = get_feeds_range_big_datetime(
request) #期間範囲指定の大きい方(新しい方)。この時間を含む
range_small_datetime = get_feeds_range_small_datetime(
request) #期間範囲指定の小さい方(古い方)。この時間を含む
user_id = get_feeds_user_id(request)
instance = get_feeds_instance(request)
content = get_feeds_content(request)
#index は 0 開始
index = get_feeds_index(request)
size = get_feeds_size(request) #指定なし時は size = -1
#返却条件は SNS に返却 かつ version 2.0 / 2.1 以外
QQ = Q(is_post_sns__ne=False) & Q(version__ne='2.0') & Q(
version__ne='2.1')
if last_feed is not None:
#last_feed の指定があった場合 (last_feed より新しい投稿を返却)
QQ &= Q(produced__gt=last_feed)
else:
#last_feed の指定がない場合 (start_timeを最新として古い投稿を返却)
QQ &= Q(produced__lt=start_time)
#range_big_datetime 指定あり
#最大より古く(<=)
if range_big_datetime is not None:
QQ &= Q(produced__lte=range_big_datetime)
#range_small_datetime 指定あり
#最小より新しい(>=)
if range_small_datetime is not None:
QQ &= Q(produced__gte=range_small_datetime)
#user_id が指定の場合はその user_id の投稿のみを抽出
from ctirs.models.rs.models import STIPUser
if user_id is not None:
stip_user = STIPUser.objects.get(id=user_id)
QQ &= (
#SNS 産 STIX なら instance 名とユーザ名が一致した
(Q(is_created_by_sns=True)
& Q(sns_user_name=stip_user.username)
& Q(sns_instance=instance)) |
#SNS 産以外の STIX なら ユーザ名が一致した
(Q(is_created_by_sns__ne=True)
& Q(sns_user_name=stip_user.username)))
else:
#未定の場合は存在している STIP_USER だけ有効
stip_users = STIPUser.objects.only('id')
stip_user_list = []
for stip_user in stip_users:
stip_user_list.append(stip_user.id)
stip_users_vias = Vias.objects.filter(uploader__in=stip_user_list)
QQ & Q(via__in=stip_users_vias)
#count = 0
l = []
stix_files = StixFiles.objects(QQ).order_by('-produced').skip(
index).timeout(False)
#サイズ指定がある場合は limit 設定
if size != -1:
stix_files = stix_files.limit(size)
#指定された数値分 StixFiles から取得する(produced順にソートする かつ is_post_sns が False以外 かつ version が 2.0 以外)
#for stix_file in StixFiles.objects(QQ).order_by('-produced').all().timeout(False):
for stix_file in stix_files:
l.append(
get_return_dictionary_from_stix_file_document(stix_file,
content=content))
r = {}
r['feeds'] = l
return JsonResponse(r, safe=False)
except Exception as e:
import traceback
traceback.print_exc()
return error(e)
def query(request):
try:
# GET 以外はエラー
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
# 引数取得
size = get_feeds_size(request) # 指定なし時は size = -1
query_string = get_query_query_string(request)
# query_string 未指定時は空リスト返却
if query_string is None:
r = {}
r['feeds'] = []
return JsonResponse(r, safe=False)
# 返却条件は SNS に返却 かつ version 2.0 以外
QQ = Q(is_post_sns__ne=False) & Q(version__ne='2.0')
# 空白スペース区切りで分割
query_strings = query_string.split(' ')
# 空白スペース区切りで検索文字列が指定されていない場合(検索対象: 投稿/タイトル/ユーザ名/スクリーン名)(大文字小文字区別せず)
if len(query_strings) == 1:
QQ &= (Q(package_name__icontains=query_strings[0])
| Q(post__icontains=query_strings[0])
| Q(sns_user_name__icontains=query_strings[0])
| Q(sns_screen_name__icontains=query_strings[0]))
else:
f_flag = True
for q in query_strings:
if f_flag:
# 空白スペース区切りの場合(検索対象: 投稿/タイトル/ユーザ名/スクリーン名)(大文字小文字区別せず)
query = Q(package_name__icontains=q) | Q(
post__icontains=q) | Q(sns_user_name__icontains=q) | Q(
sns_screen_name__icontains=q)
f_flag = False
else:
query &= Q(package_name__icontains=q) | Q(
post__icontains=q) | Q(sns_user_name__icontains=q) | Q(
sns_screen_name__icontains=q)
QQ &= (query)
stix_files = set([])
# Query
for stix_file in StixFiles.objects(QQ).order_by('-produced').timeout(
False):
stix_files.add(stix_file)
# ObservableCache から query_string を含む StixFile の set を作成
for observable_cache in ObservableCaches.objects.filter(
Q(value__icontains=query_string)):
stix_file = StixFiles.objects.get(
package_id=observable_cache.package_id)
stix_files.add(stix_file)
# 時間でソートする
stix_files = sorted(list(stix_files),
key=lambda s: s.produced,
reverse=True)
# サイズ指定がある場合は上位から指定sizeを取得
if size != -1:
stix_files = stix_files[:size]
# 返却データ作成
feeds_list = []
for stix_file in stix_files:
feeds_list.append(
get_return_dictionary_from_stix_file_document(stix_file,
content=content))
r = {}
r['feeds'] = feeds_list
return JsonResponse(r, safe=False)
except Exception as e:
import traceback
traceback.print_exc()
return error(e)
def feeds(request):
try:
# GET 以外はエラー
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
# 引数取得
start_time = get_feeds_start_time(request)
last_feed = get_feeds_last_feed(request)
range_big_datetime = get_feeds_range_big_datetime(
request) # 期間範囲指定の大きい方(新しい方)。この時間を含む
range_small_datetime = get_feeds_range_small_datetime(
request) # 期間範囲指定の小さい方(古い方)。この時間を含む
user_id = get_feeds_user_id(request)
instance = get_feeds_instance(request)
content = get_feeds_content(request)
query_string = request.GET.get(key='query_string', default=None)
# index は 0 開始
index = get_feeds_index(request)
size = get_feeds_size(request) # 指定なし時は size = -1
# 返却条件は SNS に返却
QQ = Q(is_post_sns__ne=False)
if last_feed is not None:
# last_feed の指定があった場合 (last_feed より新しい投稿を返却)
QQ &= Q(produced__gt=last_feed)
else:
# last_feed の指定がない場合 (start_timeを最新として古い投稿を返却)
QQ &= Q(produced__lte=start_time)
# range_big_datetime 指定あり
# 最大より古く(<=)
if range_big_datetime is not None:
QQ &= Q(produced__lte=range_big_datetime)
# range_small_datetime 指定あり
# 最小より新しい(>=)
if range_small_datetime is not None:
QQ &= Q(produced__gte=range_small_datetime)
# package_name か post に query_string が含まれているか (大文字小文字区別せず)
if query_string is not None:
# 空白スペース区切りで分割
query_strings = query_string.split(' ')
# 空白スペース区切りで検索文字列が指定されていない場合(検索対象: 投稿/タイトル/ユーザ名/スクリーン名)
if len(query_strings) == 1:
QQ &= (Q(package_name__icontains=query_strings[0])
| Q(post__icontains=query_strings[0])
| Q(sns_user_name__icontains=query_strings[0])
| Q(sns_screen_name__icontains=query_strings[0]))
else:
f_flag = True
for q in query_strings:
if f_flag:
# 空白スペース区切りの場合(検索対象: 投稿/タイトル/ユーザ名/スクリーン名)
query = Q(package_name__icontains=q) | Q(
post__icontains=q) | Q(
sns_user_name__icontains=q) | Q(
sns_screen_name__icontains=q)
f_flag = False
else:
query &= Q(package_name__icontains=q) | Q(
post__icontains=q) | Q(
sns_user_name__icontains=q) | Q(
sns_screen_name__icontains=q)
QQ &= (query)
# user_id が指定の場合はその user_id の投稿のみを抽出
from ctirs.models.rs.models import STIPUser
if user_id is not None:
stip_user = STIPUser.objects.get(id=user_id)
QQ &= (
# SNS 産 STIX なら instance 名とユーザ名が一致した
(Q(is_created_by_sns=True)
& Q(sns_user_name=stip_user.username)
& Q(sns_instance=instance)) |
# SNS 産以外の STIX なら ユーザ名が一致した
(Q(is_created_by_sns__ne=True)
& Q(sns_user_name=stip_user.username)))
else:
# 未定の場合は存在している STIP_USER だけ有効
stip_users = STIPUser.objects.only('id')
stip_user_list = []
for stip_user in stip_users:
stip_user_list.append(stip_user.id)
stip_users_vias = Vias.objects.filter(uploader__in=stip_user_list)
QQ & Q(via__in=stip_users_vias)
# count = 0
feeds_list = []
stix_files = StixFiles.objects(QQ).order_by('-produced').skip(
index).timeout(False)
# サイズ指定がある場合は limit 設定
if size != -1:
stix_files = stix_files.limit(size)
# 指定された数値分 StixFiles から取得する(produced順にソートする かつ is_post_sns が False以外 かつ version が 2.0 以外)
# for stix_file in StixFiles.objects(QQ).order_by('-produced').all().timeout(False):
for stix_file in stix_files:
feeds_list.append(
get_return_dictionary_from_stix_file_document(stix_file,
content=content))
r = {}
r['feeds'] = feeds_list
return JsonResponse(r, safe=False)
except Exception as e:
import traceback
traceback.print_exc()
return error(e)
def feeds(request):
try:
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
start_time = get_feeds_start_time(request)
last_feed = get_feeds_last_feed(request)
range_big_datetime = get_feeds_range_big_datetime(request)
range_small_datetime = get_feeds_range_small_datetime(request)
user_id = get_feeds_user_id(request)
instance = get_feeds_instance(request)
content = get_feeds_content(request)
query_string = request.GET.get(key='query_string', default=None)
index = get_feeds_index(request)
size = get_feeds_size(request)
QQ = Q(is_post_sns__ne=False)
if last_feed is not None:
QQ &= Q(produced__gt=last_feed)
else:
QQ &= Q(produced__lte=start_time)
if range_big_datetime is not None:
QQ &= Q(produced__lte=range_big_datetime)
if range_small_datetime is not None:
QQ &= Q(produced__gte=range_small_datetime)
if query_string is not None:
query_strings = re.split('[ ]', query_string)
if len(query_strings) == 1:
if check_symbols(query_strings[0]) and tag.is_tag(
query_strings[0]):
QQ &= Q(sns_lower_tags__in=[query_strings[0].lower()])
else:
QQ &= Q(package_name__icontains=query_strings[0]) | Q(
post__icontains=query_strings[0]) | Q(
sns_user_name__icontains=query_strings[0]) | Q(
sns_screen_name__icontains=query_strings[0])
else:
f_flag = True
for q in query_strings:
if f_flag:
if check_symbols(q) and tag.is_tag(q):
query = Q(sns_lower_tags__in=[q.lower()])
else:
query = Q(package_name__icontains=q) | Q(
post__icontains=q) | Q(
sns_user_name__icontains=q) | Q(
sns_screen_name__icontains=q)
f_flag = False
else:
if check_symbols(q) and tag.is_tag(q):
query &= Q(sns_lower_tags__in=[q.lower()])
else:
query &= Q(package_name__icontains=q) | Q(
post__icontains=q) | Q(
sns_user_name__icontains=q) | Q(
sns_screen_name__icontains=q)
QQ &= query
from ctirs.models.rs.models import STIPUser
if user_id is not None:
stip_user = STIPUser.objects.get(id=user_id)
QQ &= ((Q(is_created_by_sns=True)
& Q(sns_user_name=stip_user.username)
& Q(sns_instance=instance))
| (Q(is_created_by_sns__ne=True)
& Q(sns_user_name=stip_user.username)))
else:
stip_users = STIPUser.objects.only('id')
stip_user_list = []
for stip_user in stip_users:
stip_user_list.append(stip_user.id)
stip_users_vias = Vias.objects.filter(uploader__in=stip_user_list)
QQ & Q(via__in=stip_users_vias)
feeds_list = []
stix_files = StixFiles.objects(QQ).order_by('-produced').skip(
index).timeout(False)
if size != -1:
stix_files = stix_files.limit(size)
for stix_file in stix_files:
feeds_list.append(
get_return_dictionary_from_stix_file_document(stix_file,
content=content))
r = {}
r['feeds'] = feeds_list
return JsonResponse(r, safe=False)
except Exception as e:
import traceback
traceback.print_exc()
return error(e)
def query(request):
try:
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
size = get_feeds_size(request)
query_string = get_query_query_string(request)
if query_string is None:
r = {}
r['feeds'] = []
return JsonResponse(r, safe=False)
QQ = Q(is_post_sns__ne=False) & Q(version__ne='2.0')
query_strings = re.split('[ ]', query_string)
if len(query_strings) == 1:
if check_symbols(query_strings[0]) and tag.is_tag(
query_strings[0]):
QQ &= Q(sns_lower_tags__in=[query_strings[0].lower()])
else:
QQ &= Q(package_name__icontains=query_strings[0]) | Q(
post__icontains=query_strings[0]) | Q(
sns_user_name__icontains=query_strings[0]) | Q(
sns_screen_name__icontains=query_strings[0])
else:
f_flag = True
for q in query_strings:
if f_flag:
if check_symbols(q) and tag.is_tag(q):
query = Q(sns_lower_tags__in=[q.lower()])
else:
query = Q(package_name__icontains=q) | Q(
post__icontains=q) | Q(
sns_user_name__icontains=q) | Q(
sns_screen_name__icontains=q)
f_flag = False
else:
if check_symbols(q) and tag.is_tag(q):
query &= Q(sns_lower_tags__in=[q.lower()])
else:
query &= Q(package_name__icontains=q) | Q(
post__icontains=q) | Q(
sns_user_name__icontains=q) | Q(
sns_screen_name__icontains=q)
QQ &= query
stix_files = set([])
for stix_file in StixFiles.objects(QQ).order_by('-produced').timeout(
False):
stix_files.add(stix_file)
for observable_cache in ObservableCaches.objects.filter(
Q(value__icontains=query_string)):
stix_file = StixFiles.objects.get(
package_id=observable_cache.package_id)
stix_files.add(stix_file)
stix_files = sorted(list(stix_files),
key=lambda s: s.produced,
reverse=True)
if size != -1:
stix_files = stix_files[:size]
feeds_list = []
for stix_file in stix_files:
feeds_list.append(
get_return_dictionary_from_stix_file_document(stix_file,
content=content))
r = {}
r['feeds'] = feeds_list
return JsonResponse(r, safe=False)
except Exception as e:
import traceback
traceback.print_exc()
return error(e)