def cards(): if request.method == "GET": query_string = request.args.get("q", None) hid_number = request.args.get("hid_number", None) q_object = Q() if query_string: q_object = (q_object | Q(card_number__icontains=query_string) | Q(card_category__icontains=query_string) | Q(name__icontains=query_string) | Q(job_number__icontains=query_string) | Q(department__icontains=query_string)) if hid_number: q_object = q_object | Q(hid_card_number__icontains=hid_number) offset = request.args.get("offset", 0) limit = request.args.get("limit", 50) try: cards = ( Card.objects.filter(q_object).order_by("-created_time").skip( int(offset)).limit(int(limit))) except: current_app.logger.exception("get cards failed") abort(500) else: return make_response(cards.to_json()) elif request.method == "POST": cards_list = request.json return_list = [] try: for index, card in enumerate(cards_list): if index == 0: continue if len(card) == 1: continue c1 = Card( card_number=normalize_card_number(card[0]), card_category=card[1].strip(), name=card[2].strip(), job_number=card[3].strip(), department=card[4].strip(), gender=card[5].strip(), note=card[6].strip() if card[6] else "default", ) c1.save() return_list.append(c1) except: current_app.logger.exception("post cards failed") abort(500) else: update_all_cards_to_mc_task.delay() return make_response(jsonify({"result": len(return_list)})) elif request.method == "DELETE": cards_to_delete = json.loads(request.args["delete_array"]) cards_to_delete2 = [] try: for card in cards_to_delete: card_obj = Card.objects.get(pk=card) card_2 = json.loads(card_obj.to_json()) cards_to_delete2.append(card_2) card_obj.delete() except: current_app.logger.exception("delete cards failed") abort(500) else: for card_2 in cards_to_delete2: delete_a_card_from_mc_task.delay(card_2) return make_response(jsonify({"result": len(cards_to_delete)}))
def check_user(self, username, password): if not Users.objects( Q(user_name__exact=username) & Q(password__exact=password)): return None user = Users.objects.get(user_name=username) return user
def get(self, request, format=None): # userid=int(request.GET['userId']) json_out={} data=[] tokens = decode_base64(request.META.get('HTTP_AUTHORIZATION')) # tokens = base64.b64decode(request.META.get('HTTP_AUTHORIZATION')) time_stamp = tokens[-13:-3] if abs(int(time_stamp)-int(time.time())) < 60: tokens = re.sub(r'#.*#.*','',tokens) # tokens = json_data['userid'] pld = Auth.decode_auth_token(tokens) user_id = pld['data']['id'] login_time = pld['data']['login_time'] try: # res=Hot_Topic.objects(Q(user_id=userid)&Q(_id__lte=137)) res=Hot_Topic.objects(Q(_id__lte=140)) for topic in res: temp={} temp['topicId']=topic._id temp['topicName']=topic.topic_name temp['topicKeywords']=topic.topic_kws temp['imgs']= [] if topic._id == 130: temp['imgs'] = [ "http://www.moe.edu.cn/jyb_xwfb/xw_fbh/moe_2069/xwfbh_2017n/xwfb_070621/201706/W020170621386205134556.jpg", "http://www.moe.gov.cn/jyb_xwfb/s6052/moe_838/201805/W020180503693998846906.jpg" ] elif topic._id == 131 : temp['imgs'] = [ "https://gss0.baidu.com/-vo3dSag_xI4khGko9WTAnF6hhy/zhidao/wh%3D600%2C800/sign=62b13c2e0d46f21fc9615655c6144758/cefc1e178a82b901ffebb1e6748da9773812efa5.jpg", "http://www.81.cn/hkht/attachement/jpg/site351/20150921/18037331c3e11769f42e0e.jpg" ] elif topic._id == 132: temp['imgs'] = [ "http://img.caixin.com/2017-05-28/1495931830739657_480_320.jpg", "http://img.caixin.com/2017-05-28/1495931818880517.jpg" ] elif topic._id == 133 : temp['imgs'] = [ "https://timgsa.baidu.com/timg?image&quality=80&size=b9999_10000&sec=1525780379793&di=a3a5919e75d989bd440cbaf220285986&imgtype=0&src=http%3A%2F%2Fn.sinaimg.cn%2Fnews%2F20170111%2Fd009-fxzkssy1982912.jpg", "https://timgsa.baidu.com/timg?image&quality=80&size=b9999_10000&sec=1525780367057&di=4fa93c0e58282e8b922ab9164582251d&imgtype=0&src=http%3A%2F%2Fi2.sinaimg.cn%2Fdy%2Fzl%2Fzatan%2F2015-12-17%2FU12776P1T940D5128F24202DT20151217094145.jpg" ] elif topic._id == 134 : temp['imgs'] = [ "http://img1.cache.netease.com/catchpic/8/86/866900747709FBDAD41BC788DC166E85.jpg", "http://www.cs090.com/uploads/userup/333977/2015/1451104091-62H352.jpg" ] elif topic._id == 135 : temp['imgs'] = [ "https://goss3.vcg.com/creative/vcg/800/version23/VCG21a809942a0.jpg", "http://www.gsstc.gov.cn/pic/2014_04/%7B194C4BEF-CA53-41CB-17F9-247BE28F05BE%7D.JPG" ] elif topic._id == 137 : temp['imgs'] = [ "http://www.17ok.com/files_root/upload_file/media/images/201310/20131014112307322.jpg", "http://jjckb.xinhuanet.com/images/2014-12/20/xin_111211201146295262568.jpg" ] elif topic._id == 138 : temp['imgs'] = [ "http://image.sinajs.cn/newchart/hk_stock/min_660/hsi.gif", "http://www.cash.com.hk/en/img/banner.jpg" ] elif topic._id == 139 : temp['imgs'] = [ "http://campusrd.zhaopin.com/CompanyLogo/20141018/492BFE08208B4E1E9590F1CCE0784F8A.jpg", "https://timgsa.baidu.com/timg?image&quality=80&size=b9999_10000&sec=1525780345536&di=d559538bb39e2c9db59177e851842025&imgtype=0&src=http%3A%2F%2Fcdns.soufunimg.com%2Fimgs%2Fhouse%2F2015_08%2F10%2Fhefei%2F1439169810477_000.jpg" ] elif topic._id == 136 : temp['imgs'] = [ "http://finance.ce.cn/rolling/201305/13/W020130615723679354246.jpg", "http://www.0938f.com/userfiles/image/20150924/24100931069729c1de2991.jpg" ] else : pass # res = Post_News.objects(hot_topic_id=topic._id) # for post_res in res : # if len(post_res.img_url) > 5: # temp['imgs'].append(post_res.img_url) # else : # pass # try: # temp['imgs'] = random.sample(temp['imgs'],5) # except: # temp['imgs'] = temp['imgs'] temp['summary']=topic.summary data.append(temp) json_out['code']=0 json_out['success']=True json_out['data']=data except: traceback.print_exc() json_out['code']=1 json_out['success']=False json_out['data']={} return HttpResponse(json.dumps(json_out, cls=MyEncoder),content_type="application/json") else: json_out['code'] = 1 json_out['data'] = '认证失败' json_out['success'] = False return HttpResponse(json.dumps(json_out, cls=MyEncoder),content_type="application/json")
def cardtests(): q_object = Q() datetime_from = request.args.get("datetime_from", None) datetime_to = request.args.get("datetime_to", None) # '2018-07-20T07:15:00.000Z' job_number = request.args.get("job_number", None) card_number = request.args.get("card_number", None) department = request.args.get("department", None) is_downloading_excel = request.args.get("is_downloading_excel", None) name = request.args.get("name", None) mc_id = request.args.get("mc_id", None) card_cat = request.args.get("card_cat", None) hid_number = request.args.get("hid_number", None) if datetime_from: datetime_from = datetime.datetime.strptime( datetime_from, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=datetime.timezone.utc) q_object = q_object & Q(test_datetime__gte=datetime_from) if datetime_to: datetime_to = datetime.datetime.strptime( datetime_to, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=datetime.timezone.utc) q_object = q_object & Q(test_datetime__lte=datetime_to) if hid_number: card_number = hid_to_normal(hid_number) q_object = q_object & Q(card_number__icontains=card_number) elif card_number: card_number = normalize_card_number(card_number) q_object = q_object & Q(card_number__icontains=card_number) if job_number: cards = Card.objects.filter(job_number__icontains=job_number) q_object = q_object & Q( card_number__in=[card.card_number for card in cards]) if department: cards = Card.objects.filter(department__icontains=department) q_object = q_object & Q( card_number__in=[card.card_number for card in cards]) if name: cards = Card.objects.filter(name__icontains=name) q_object = q_object & Q( card_number__in=[card.card_number for card in cards]) if mc_id: q_object = q_object & Q(mc_id__icontains=mc_id) if card_cat: q_object = q_object & Q(card_category=card_cat) offset = request.args.get("offset", 0) limit = request.args.get("limit", 50) try: if is_downloading_excel: results = [[ "log_id", "卡片编号", "卡片号码", "卡片分类", "进出标志", "mc id", "测试时间", "测试结果", "是否测试", "手测试值(KΩ)", "左脚测试值(KΩ)", "右脚测试值(KΩ)", "erg后数值", "rsg", "姓名", "工号", "HID 卡号", ]] logs = CardTest.objects.filter(q_object).order_by("-test_datetime") for log in logs: card_category = "" if log["card_category"] == "0": card_category = "VIP" if log["card_category"] == "1": card_category = "只测手" if log["card_category"] == "2": card_category = "只测脚" if log["card_category"] == "3": card_category = "手脚都测" in_out_symbol = "" if log["in_out_symbol"] == "0": in_out_symbol = "出" if log["in_out_symbol"] == "1": in_out_symbol = "进" # 获得系统时区 system_config_timezone = int( SystemConfig.objects.get().timezone) local_tz = datetime.timezone( datetime.timedelta(hours=system_config_timezone)) test_datetime = (log["test_datetime"].replace( tzinfo=datetime.timezone.utc).astimezone( local_tz).isoformat()) test_result = "" if log["test_result"] == "0": test_result = "不通过" if log["test_result"] == "1": test_result = "通过" is_tested = "" if log["is_tested"] == "0": is_tested = "不测试" if log["is_tested"] == "1": is_tested = "测试" name = "" job_number = "" try: card = Card.objects.filter( card_number=log["card_number"]).first() name = card["name"] job_number = card["job_number"] except: pass results.append([ log["log_id"], log["card_counter"], log["card_number"], card_category, in_out_symbol, log["mc_id"], test_datetime, test_result, is_tested, log["hand"], log["left_foot"], log["right_foot"], log["after_erg"], log["rsg"], name, job_number, hid_number, ]) return excel.make_response_from_array(results, "xlsx") cards = ( CardTest.objects.filter(q_object).order_by("-test_datetime").skip( int(offset)).limit(int(limit))) return cards.to_json(), {"Content-Type": "application/json"} except: current_app.logger.exception("get cardtests failed") abort(500)
def package_list_for_sharing_table(request): try: if request.method != 'GET': return HttpResponseNotAllowed(['GET']) # 認証する user = authentication(request) if user is None: return error( Exception('You have no permission for this operation.')) # ajax parameter取得 # 表示する長さ iDisplayLength = int(request.GET['iDisplayLength']) # 表示開始位置インデックス iDisplayStart = int(request.GET['iDisplayStart']) # 検索文字列 sSearch = request.GET['sSearch'] # ソートする列 sort_col = int(request.GET['iSortCol']) # ソート順番 (desc指定で降順) sort_dir = request.GET['sSortDir'] order_query = None SORT_INDEX_PACKAGE_NAME = 3 # pakcage_name if sort_col == SORT_INDEX_PACKAGE_NAME: order_query = 'package_name' # 昇順/降順 if order_query is not None: # descが降順 if sort_dir == 'desc': order_query = '-' + order_query # 検索対象のコミュニティリストを検索 community_objects = Communities.objects.filter(name__icontains=sSearch) # 検索 objects = StixFiles.objects.filter( Q(package_name__icontains=sSearch) | Q(input_community__in=community_objects)) \ .order_by(order_query) objects = objects.filter(Q(is_post_sns__ne=False)) # 検索結果から表示範囲のデータを抽出する data = [] for d in objects[iDisplayStart:(iDisplayStart + iDisplayLength)]: r = {} r['comment'] = d.comment r['package_name'] = d.package_name r['package_id'] = d.package_id try: r['input_community'] = d.input_community.name except BaseException: r['input_community'] = '' data.append(r) # response data 作成 r_data = {} r_data['iTotalRecords'] = StixFiles.objects.count() r_data['iTotalDisplayRecords'] = objects.count() r_data['data'] = data resp = get_normal_response_json() resp['data'] = r_data return JsonResponse(resp) except Exception as e: traceback.print_exc() return error(e)
def get(self, request, format=None): # userid=int(request.GET['userId']) topicid=int(request.GET['topicId']) json_out={} data={} tokens = decode_base64(request.META.get('HTTP_AUTHORIZATION')) # tokens = base64.b64decode(request.META.get('HTTP_AUTHORIZATION')) time_stamp = tokens[-13:-3] if abs(int(time_stamp)-int(time.time())) < 60: tokens = re.sub(r'#.*#.*','',tokens) # tokens = json_data['userid'] pld = Auth.decode_auth_token(tokens) user_id = pld['data']['id'] login_time = pld['data']['login_time'] try: year_now=date.today().year month_now=date.today().month day_now=date.today().day datatype_list = Datatype_name.objects month_find=0 year_find=0 date_find_min=None date_find_max=None if month_now<3: month_find=month_now-2+12 year_find=year_now-1 else: month_find=month_now-2 year_find=year_now date_find_min=datetime.datetime.combine(date(year_find, month_find, 1), datetime.time.min) date_find_max=datetime.datetime.combine(date(year_now, month_now, day_now), datetime.time.max) topics=Hot_Topic.objects(_id=topicid) topics_name = [] topic_relat_list = [] topic_relations = Topic_Relation.objects(Q(_id=topicid)) # print len(topic_relations) for topic_relat in topic_relations: index = topic_relat._id - 130 for ind,val in enumerate(topic_relat.topic_relat) : # print '222' if index == ind : continue triple_group = {} triple_group['source'] = index triple_group['target'] = ind triple_group['weight'] = val # triple_group = [index,ind,val] topic_relat_list.append(triple_group) all_topics = Hot_Topic.objects(Q(_id__gte=130)&Q(_id__lte=139)) print len(all_topics) for topic in all_topics: topic_obj = {} topic_obj['id'] = topic._id topic_obj['topic_name'] = topic.topic_name topic_obj['topic_kws'] = topic.topic_kws topic_obj['user_id'] = topic.user_id topic_obj['user_name'] = topic.user_name topic_obj['summary'] = topic.summary topics_name.append(topic_obj) for topic in topics: data['topicId']=topic._id data['topic_index']=topic._id-130 res_evos=Topic_evolution.objects(Q(topic_id=str(topicid))) print '11' # count = 0 data_list=[] for item in res_evos: # count += 1 # if count == 500 : # break # else : # pass temp={} # temp['id']=str(item._id) temp['time']=item.time temp['number']=item.number temp['topic_id']=item.topic_id temp['word']=item.word # temp['topic']=item.topic[0] data_list.append(temp) data['topic_info']=data_list img_list = [] data['topicName']=topic.topic_name data['topic_kws']={} data['all_topic']=topics_name data['topic_relation']=topic_relat_list for kw in topic.topic_kws: data['topic_kws'][kw] = random.randint(1,100) # data['topic_kws']=topic.topic_kws # click_num=0 postdata=[] for data_type in datatype_list: print '11111111' res1=Post_News.objects(Q(hot_topic_id=topicid)&Q(data_type=data_type.data_type)).only('pt_time','site_id','data_type','comm_num','img_url','title','content') # res1=Post_News.objects(Q(hot_topic_id=topicid)).only('pt_time','site_id','data_type','comm_num') # click_num=res1.sum('comm_num') print len(res1) # click_num=click_num+res1.sum('comm_num') for post in res1: temp={} temp['postTime']=post.pt_time temp['site_id']=post.site_id temp['title']=post.title if len(post.img_url) > 5: img_list.append(post.img_url) else : pass temp['content']=post.content[:100] temp['dataType']=post.data_type temp['site_name']=sitedict[int(post.site_id)] temp['dataTypeName']=datatypedict[int(post.data_type)] postdata.append(temp) # summa = 0 # for post in res1[:200]: # # if summa > 100 : # # continue # # else: # # pass # # summa +=1 # temp={} # temp['postTime']=post.pt_time # temp['site_id']=post.site_id # temp['title']=post.title # if len(post.img_url) > 5: # img_list.append(post.img_url) # else : # pass # temp['content']=post.content # temp['dataType']=post.data_type # temp['site_name']=sitedict[int(post.site_id)] # temp['dataTypeName']=datatypedict[int(post.data_type)] # postdata.append(temp) try: data['img_url'] = random.sample(img_list,5) except: data['img_url'] = img_list data['clickNums']=click_num data['postData']=postdata json_out['code']=0 json_out['success']=True json_out['data']=data except: traceback.print_exc() json_out['code']=1 json_out['success']=False json_out['data']={} return HttpResponse(json.dumps(json_out, cls=MyEncoder),content_type="application/json") else: json_out['code'] = 1 json_out['data'] = '认证失败' json_out['success'] = False return HttpResponse(json.dumps(json_out, cls=MyEncoder),content_type="application/json")
def _get_observables_cashes_query(pattern, package_id): q = Q() # pattern 文字列から種別と値を正規表現で抽出し、 OR 検索で連結する for v in url_pattern.findall(pattern): q |= (Q(package_id__ne=package_id) & Q(type__exact='uri') & Q(value__exact=v)) for v in ipv4_pattern.findall(pattern): q |= (Q(package_id__ne=package_id) & Q(type__exact='ipv4') & Q(value__exact=v)) for v in domain_name_pattern.findall(pattern): q |= (Q(package_id__ne=package_id) & Q(type__exact='domain_name') & Q(value__exact=v)) for v in md5_pattern.findall(pattern): q |= (Q(package_id__ne=package_id) & Q(type__exact='md5') & Q(value__exact=v)) for v in sha1_pattern.findall(pattern): q |= (Q(package_id__ne=package_id) & Q(type__exact='sha1') & Q(value__exact=v)) for v in sha256_pattern.findall(pattern): q |= (Q(package_id__ne=package_id) & Q(type__exact='sha256') & Q(value__exact=v)) for v in sha512_pattern.findall(pattern): q |= (Q(package_id__ne=package_id) & Q(type__exact='sha512') & Q(value__exact=v)) return q
def post(self, request, format=None): recommendFilter = request.data # Realiza la consulta con los filtrosy ordenador por la calificación en orden ascendente recipesQuery = Recipes.objects( Q(calories__lt=recommendFilter["calories"]) & Q(protein__lt=recommendFilter["protein"]) & Q(sodium__lt=recommendFilter["sodium"]) & Q(ingredients__nin=recommendFilter["ingredients"])).order_by( "-rating")[:5000] #Mirar como limitar la consulta # recipesQuery = Recipes.objects( # Q(calories__lt=recommendFilter["calories"]) & Q(protein__lt=recommendFilter["protein"]) & # Q(sodium__lt=recommendFilter["sodium"]) # ).order_by("-rating") serializer = RecipeSerializer(recipesQuery, many=True) # recipes = recipesQuery.data # # if recipes.__len__() == 0: #No tiene característica __len__()? # return Response(status=status.HTTP_404_NOT_FOUND) # elif recipes.__len__() > 1: # serializer = RecipeSerializer(recipes, many=True) # else: # serializer = RecipeSerializer(recipes, many=False) # Obtenemos el json # response = {"recipes": serializer.data} # Obtenemos el DataFrame con los datos serializados dataframe = pd.DataFrame.from_dict(serializer.data) # Obtenemos los datos que queremos entrenar train = dataframe[["calories", "protein", "sodium"]] data_train, test_train = train_test_split(train, test_size=0.2, random_state=42) # Entrenamos los datos k = 10 # Número de muestras que nos va a devolver neig = neighbors.NearestNeighbors( k, 'cosine') # Aplicamos el algoritmo de vecinos más cercanos neig.fit(data_train) #Entrenamos los datos top_k_distances, top_k_items = neig.kneighbors( data_train, return_distance=True) # Obtenemos los vecinos más cercanos top = [] for item in top_k_items: top.extend(item) # a = top_k_items[0] #Escogemos la primera opción recipes = dataframe.iloc[top] # Obtenemos los datos del dataframe response = {"recipes": recipes.T} # Creamos un JSON # Escogemos las recetas recomendadas y creamos dietas teniendo en cuenta los datos dados por el usuario diets = [] diet = Diet() day = Day() # diet = Diet() # diet.days = [] countCalories = 0.0 countSodium = 0.0 countProteins = 0.0 morning = 0 count = 0 for i in recipes[[ "id", "title", "calories", "protein", "sodium", "rating", "ingredients", "directions", "categories", "description" ]].values: # Obtener para el resto de valores cal = float(i[2]) prot = float(i[3]) sod = float(i[4]) rate = float(i[5]) if diets.__len__() == 10: break if cal < recommendFilter["calories"] and prot < recommendFilter["protein"]\ and sod < recommendFilter["sodium"]\ and (countCalories + cal) <= recommendFilter["calories"]\ and (countProteins + prot) <= recommendFilter["protein"]\ and (countSodium + sod) <= recommendFilter["sodium"]\ and diet.days.__len__() <= 7: recipe = Rec(i[1], cal, prot, sod, rate, i[6], i[7], i[8], i[9]) countCalories += cal countProteins += prot countSodium += sod if morning == 0: day.recipes.append(recipe.__dict__) morning = 1 elif morning == 1: day.name = "Día " + (count + 1).__str__() day.recipes.append(recipe.__dict__) morning = 0 count = count + 1 diet.days.append(day.__dict__) day = Day() if diet.days.__len__() >= 7: diets.append(diet.__dict__) diet = Diet() countCalories = 0.0 countSodium = 0.0 countProteins = 0.0 morning = 0 count = 0 response = {"diet": diets} # for i in diets: # print(i.__dict__) # print(i) # print(i.__str__()) # # response = json.dumps([di.__dict__ for di in diets]) # print(response) # return JsonResponse() return Response(response, status=status.HTTP_200_OK)
ret = {} try: sceneInfoResult = sceneInfo.objects( Q(userId=id) & Q(triggerIntent__contains=intentName)).first() if sceneInfoResult is not None: ret['isTrigger'] = True ret['sceneName'] = sceneInfoResult['sceneName'] else: ret['isTrigger'] = False ret['sceneName'] = "" except: traceback.print_exc() ret['isTrigger'] = False ret['sceneName'] = "" return ret if __name__ == "__main__": userId = "user111" sceneName = "SCENE_ADJUST_QUOTA" sceneResult = sceneInfo.objects(Q(userId=userId) & Q(sceneName=sceneName)).first() if (sceneResult is not None): sceneResultDict = sceneResult.to_mongo().to_dict() triggerIntent = sceneResultDict['triggerIntent'] flowConfig = sceneResultDict['flowConfig'] intentInfo = {} for key in triggerIntent: intentInfo[key] = LoadData().loadIntentData(intentName=key, userId=userId)
def _get_exact_matched_info(package_id): ret_observable_cashes = [] # match 検索対象の cache list cache_collections = [ ObservableCaches, # STIX 1.x/2.x の Observables ExploitTargetCaches, ] for cache_collection in cache_collections: caches = cache_collection.objects.filter(package_id=package_id) for cache in caches: # type,valueが一緒で、検索 package_idと異なる Cache を検索 (observables 間検索) # Obaservable の値がない場合はスキップする if cache.value is None: continue exacts = cache_collection.objects.filter( Q(type__exact=cache.type) & Q(value__exact=cache.value) & Q(package_id__ne=package_id)) ret_observable_cashes.extend(exacts) # ObservableCaches の検索の場合はさらに その Observable が STIX v2 の indicator pattern 文字列にヒットするか判定する if isinstance(cache, ObservableCaches): # Observable と STIX 2.x の indicators 間 indicator_v2_cachses = IndicatorV2Caches.objects.filter( Q(package_id__ne=package_id)) for indicator_v2_cache in indicator_v2_cachses: observed_data = _get_observed_data(cache) matches = matcher.match(indicator_v2_cache.pattern, [observed_data]) if len(matches) != 0: indicator_v2_cache.type = cache.type indicator_v2_cache.value = cache.value # start_node の検索先は ObservableCaches or ExploitTargetCaches indicator_v2_cache.start_collection = cache_collection # 格納する end_info は (指定の package_id) の ObservableCache に該当する IndicatorV2Cache ret_observable_cashes.append(indicator_v2_cache) # STIX2 indicator cache の matching indicator_v2_caches = IndicatorV2Caches.objects.filter( package_id=package_id) for indicator_v2_cache in indicator_v2_caches: # 対象の IndicatorV2Caches に Match する (Package_id は別の) ObservableCaches を検索する query_set = _get_observables_cashes_query(indicator_v2_cache.pattern, package_id) caches = ObservableCaches.objects.filter(query_set) for cache in caches: observed_data = _get_observed_data(cache) matches = matcher.match(indicator_v2_cache.pattern, [observed_data]) if len(matches) != 0: # start_info は IndicatorV2Caches から検索 cache.start_collection = IndicatorV2Caches cache.pattern = indicator_v2_cache.pattern # 格納する end_info は (指定の package_id) の Indicator v2 にマッチする ObservableCache ret_observable_cashes.append(cache) # 自分以外の PackageID で STIX2 indicator pattern を検索する exacts = IndicatorV2Caches.objects.filter( Q(pattern__exact=indicator_v2_cache.pattern) & Q(package_id__ne=package_id)) for exact in exacts: exact.cache_list = IndicatorV2Caches # start_node の検索先は ObservableCaches or ExploitTargetCaches exact.start_collection = IndicatorV2Caches # 格納するのは (指定の package_id) の Indicator v2 にマッチする (指定の package_id 以外に)IndicatorV2Cache ret_observable_cashes.append(exact) return ret_observable_cashes
def advanced_search(query): # lexer toks = [] while len(query) > 0: incr = 0 if re.search(r'^ ', query): incr = 1 elif re.search(r'^AND', query): incr = 3 toks.append('AND') elif re.search(r'^NOT', query): incr = 3 toks.append('NOT') else: next_term = re.search(r' +(AND|NOT) +', query) if next_term: # gets next term based on everything before next AND or NOT toks.append(query[:next_term.span()[0]]) incr = next_term.span()[0] else: # if next term is at end of query term = re.search(r'^\S.*', query) if term: toks.append(term.group()) incr = len(term.group()) query = query[incr:] # parser and_count = 0 ands = None nots = None # validates and preprocesses query if toks[0] == 'AND': return [] if toks[0] != 'NOT': ands = Q(tag=re.compile("^" + toks[0] + "$", re.IGNORECASE)) toks = toks[1:] and_count += 1 if len(toks) % 2 == 1: return [] while len(toks) > 0: if toks[0] == 'AND': q = Q(tag=re.compile("^" + toks[1] + "$", re.IGNORECASE)) if ands == None: ands = q else: ands = ands | q and_count += 1 elif toks[0] == 'NOT': q = Q(tag=re.compile("^" + toks[1] + "$", re.IGNORECASE)) if nots == None: nots = q else: nots = nots | Q(tag=re.compile("^" + toks[1] + "$", re.IGNORECASE)) else: return [] toks = toks[2:] # get ids and_tags = Tag.objects(ands) if ands != None else [] not_tags = Tag.objects(nots) if nots != None else [] # if invalid tag exists in and tags if len(and_tags) != and_count: return [] # get files q = None for tag in and_tags: if q == None: q = Q(tags__contains=tag.id) else: q = q & Q(tags__contains=tag.id) for tag in not_tags: if q == None: q = Q(tags__ne=tag.id) else: q = q & Q(tags__ne=tag.id) files = File.objects(q) return files
def get(self, dataset_id): """ Endpoint called by image viewer client """ parsed_args = page_data.parse_args() per_page = parsed_args.get('limit') page = parsed_args.get('page') - 1 folder = parsed_args.get('folder') order = parsed_args.get('order') args = dict(request.args) # Check if dataset exists dataset = current_user.datasets.filter(id=dataset_id, deleted=False).first() if dataset is None: return {'message', 'Invalid dataset id'}, 400 # Make sure folder starts with is in proper format if len(folder) > 0: folder = folder[0].strip('/') + folder[1:] if folder[-1] != '/': folder = folder + '/' # Get directory directory = os.path.join(dataset.directory, folder) if not os.path.exists(directory): return {'message': 'Directory does not exist.'}, 400 # Remove parsed arguments for key in parsed_args: args.pop(key, None) # Generate query from remaining arugments query = {} for key, value in args.items(): lower = value.lower() if lower in ["true", "false"]: value = json.loads(lower) if len(lower) != 0: query[key] = value # Change category_ids__in to list if 'category_ids__in' in query.keys(): query['category_ids__in'] = [ int(x) for x in query['category_ids__in'].split(',') ] # Initialize mongo query with required elements: query_build = Q(dataset_id=dataset_id) query_build &= Q(path__startswith=directory) query_build &= Q(deleted=False) # Define query names that should use complex logic: complex_query = ['annotated', 'category_ids__in'] # Add additional 'and' arguments to mongo query that do not require complex_query logic for key in query.keys(): if key not in complex_query: query_dict = {} query_dict[key] = query[key] query_build &= Q(**query_dict) # Add additional arguments to mongo query that require more complex logic to construct if 'annotated' in query.keys(): if 'category_ids__in' in query.keys() and query['annotated']: # Only show annotated images with selected category_ids query_dict = {} query_dict['category_ids__in'] = query['category_ids__in'] query_build &= Q(**query_dict) else: # Only show non-annotated images query_dict = {} query_dict['annotated'] = query['annotated'] query_build &= Q(**query_dict) elif 'category_ids__in' in query.keys(): # Ahow annotated images with selected category_ids or non-annotated images query_dict_1 = {} query_dict_1['category_ids__in'] = query['category_ids__in'] query_dict_2 = {} query_dict_2['annotated'] = False query_build &= (Q(**query_dict_1) | Q(**query_dict_2)) # Perform mongodb query images = current_user.images \ .filter(query_build) \ .order_by(order).only('id', 'file_name', 'annotating', 'annotated', 'num_annotations', 'category_ids') total = images.count() pages = int(total / per_page) + 1 images = images.skip(page * per_page).limit(per_page) images_json = query_util.fix_ids(images) # for image in images: # image_json = query_util.fix_ids(image) # query = AnnotationModel.objects(image_id=image.id, deleted=False) # category_ids = query.distinct('category_id') # categories = CategoryModel.objects(id__in=category_ids).only('name', 'color') # image_json['annotations'] = query.count() # image_json['categories'] = query_util.fix_ids(categories) # images_json.append(image_json) subdirectories = [ f for f in sorted(os.listdir(directory)) if os.path.isdir(directory + f) and not f.startswith('.') ] categories = CategoryModel.objects(id__in=dataset.categories).only( 'id', 'name') return { "total": total, "per_page": per_page, "pages": pages, "page": page, "images": images_json, "folder": folder, "directory": directory, "dataset": query_util.fix_ids(dataset), "categories": query_util.fix_ids(categories), "subdirectories": subdirectories }
def random(self, request, *args, **kwargs): count = Place.objects.count() random = randint(0, count - 10) places = Place.objects( Q(types__contains="restaurant") | Q(types__contains="bar") | Q(types__contains="food") | Q(types__contains="bakery") | Q(types__contains="cafe") | Q(types__contains="casino") | Q(types__contains="convenience_store") | Q(types__contains="meal_delivery") | Q(types__contains="make_takeaway") | Q(types__contains="nightclub") | Q(types__contains="shopping_mall")).limit(10).skip(random) serializer = PlaceSerializer(places, many=True) return Response(serializer.data)
def dashboard_sourceData(request): if request.method == 'GET': json_out = {} main_out = {} days_num = 0 data = {} try: # sourcedata days_list = [] day = pd.Period(datetime.datetime.now(),freq='D') # logger.info(type(Topic.objects)) # Topic.objects.all()Topic.objects()返回类型相同 topic_list = Topic.objects datatype_list = Datatype_name.objects today = date.today() post_7days = Post.objects(Q(pt_time__gte=datetime.datetime.combine\ (today-datetime.timedelta(7),datetime.time.min)) & Q(pt_time__lte=datetime.datetime.combine(today, datetime.time.max))) while days_num < 7: day_str = day.strftime('%Y%m%d') day_change = today - datetime.timedelta(days=days_num) post = post_7days(Q(pt_time__gte=\ datetime.datetime.combine(day_change,datetime.time.min)) & \ Q(pt_time__lte=datetime.datetime.combine\ (day_change, datetime.time.max))) for topic in topic_list: for data in datatype_list: day_dict = {} day_dict['time'] = day_str day_dict['topic_id'] = topic._id day_dict['topic_name'] = topic.topic_name day_dict['dataTypeName'] = data.datatype_name day_dict['data_type'] = data.data_type post_datatype = post(Q(data_type=data.data_type) & Q(topic_id=topic._id)) # logger.info('post_num = ' + str(len(post_datatype))) day_dict['post_num'] = post_datatype.count() days_list.append(day_dict) for data in datatype_list: day_dict = {} day_dict['time'] = day_str day_dict['topic_id'] = 0 day_dict['topic_name'] = '' day_dict['dataTypeName'] = data.datatype_name day_dict['data_type'] = data.data_type post_datatype = post(Q(data_type=data.data_type) & Q(topic_id=0)) # logger.info('post_num = ' + str(len(post_datatype))) day_dict['post_num'] = post_datatype.count() days_list.append(day_dict) day -= 1 days_num += 1 ####### Hot for all host posts hot_dict = {} hot_posts = post_7days(Q(topic_id__ne=0) & (Q(data_type=3) | Q(data_type=2))).order_by \ ('-comm_num')[:10].only("_id", "url", \ "board", "title", "content", "pt_time", \ "img_url", "poster") # hot_poster = post_7days.only('poster').all() # logger.info("hot_poster = " + str(hot_poster.count())) hot_weibo = post_7days(Q(topic_id__ne=0) & Q(data_type=2)).order_by \ ('-comm_num')[:10].only("_id", "url", \ "board", "title", "content", "pt_time", \ "img_url") hot_dict['hotPost'] = handle_post_list(hot_posts) hot_dict['hotPoster'] = handle_poster_list(hot_posts) hot_dict['hotWeibo'] = handle_post_list(hot_weibo) # wordlist = [] # wordres=Cloud_formain.objects.only("word", "frequency") # for worditem in wordres: # temp={} # temp['word']=worditem.word # temp['weight']=worditem.frequency # wordlist.append(temp) ####### map data mapData_list = [{'id':'001', 'pro':"陕西", 'nums':52 }, { 'id':'002', 'pro':"北京", 'nums':100 }, { 'id':'003', 'pro':"上海", 'nums':60 }, { 'id':'004', 'pro':"杭州", 'nums':48 }, { 'id':'005', 'pro':"南京", 'nums':50 } ] main_out['mapData'] = mapData_list main_out['sourceData'] = days_list main_out['Hot'] = hot_dict # main_out['word_cloud'] = wordlist json_out['code'] = 0 json_out['success'] = True json_out['data'] = main_out except: traceback.print_exc() json_out['code'] = 1 json_out['data'] = {} json_out['success'] = False return HttpResponse(json.dumps(json_out, cls=MyEncoder),content_type="application/json")
def l1_info_for_l1table(request): try: if request.method != 'GET': return HttpResponseNotAllowed(['GET']) # 認証する user = authentication(request) if user is None: return error( Exception('You have no permission for this operation.')) # ajax parameter取得 # 表示する長さ iDisplayLength = int(request.GET['iDisplayLength']) # 表示開始位置インデックス iDisplayStart = int(request.GET['iDisplayStart']) # 検索文字列 sSearch = request.GET['sSearch'] # ソートする列 sort_col = int(request.GET['iSortCol']) # ソート順番 (desc指定で降順) sort_dir = request.GET['sSortDir'] # alias情報 # 存在しない場合は空としてあつかつ try: aliases_str = request.GET['aliases'] alias_lists = json.loads(aliases_str) except BaseException: alias_lists = [] order_query = None SORT_INDEX_TYPE = 0 SORT_INDEX_VALUE = 1 SORT_INDEX_PACKAGE_NAME = 2 SORT_INDEX_TILE = 3 SORT_INDEX_DESCRIPTION = 4 SORT_INDEX_TIMESTAMP = 5 # type if sort_col == SORT_INDEX_TYPE: order_query = 'type' # value elif sort_col == SORT_INDEX_VALUE: order_query = 'value' # pacakge_name elif sort_col == SORT_INDEX_PACKAGE_NAME: order_query = 'package_name' # title elif sort_col == SORT_INDEX_TILE: order_query = 'title' # description elif sort_col == SORT_INDEX_DESCRIPTION: order_query = 'description' # timestamp elif sort_col == SORT_INDEX_TIMESTAMP: order_query = 'produced' # 昇順/降順 if order_query is not None: # descが降順 if sort_dir == 'desc': order_query = '-' + order_query # query # 検索ワードをリスト化 tmp_sSearches = list(set(sSearch.split(' '))) # 空要素は取り除く if '' in tmp_sSearches: tmp_sSearches.remove('') # 検索リスト作成 sSearches = [] for item in tmp_sSearches: # まず、元の単語は追加する sSearches.append(item) # alias_lists 1つずつチェックする for alias_list in alias_lists: # 検索ワードがalias_listにあれば、そのリストに含まれるすべての単語が検索対象 if item in alias_list: sSearches.extend(alias_list) # 重複を省く sSearches = list(set(sSearches)) # Filterを作成する filters = Q() # alias含め、その文字列が含まれていたらヒットとする for sSearch in sSearches: filters = filters | Q(type__icontains=sSearch) filters = filters | Q(value__icontains=sSearch) filters = filters | Q(package_name__icontains=sSearch) filters = filters | Q(title__icontains=sSearch) filters = filters | Q(description__icontains=sSearch) # 検索 objects = ObservableCaches.objects.filter(filters).order_by( order_query) # 検索結果から表示範囲のデータを抽出する data = [] for d in objects[iDisplayStart:(iDisplayStart + iDisplayLength)]: r = {} r['type'] = d.type r['value'] = d.value r['package_name'] = d.package_name r['package_id'] = d.stix_file.package_id r['title'] = d.title r['description'] = d.description r['created'] = str(d.created) r['stix_v2'] = d.stix_file.is_stix_v2() r['observable_id'] = d.observable_id data.append(r) # response data 作成 r_data = {} r_data['iTotalRecords'] = ObservableCaches.objects.count() r_data['iTotalDisplayRecords'] = objects.count() r_data['data'] = data resp = get_normal_response_json() resp['data'] = r_data return JsonResponse(resp) except Exception as e: traceback.print_exc() return error(e)
def dashboard_sourceData_temp(request): if request.method == 'GET': json_out = {} main_out = {} days_num = 0 data = {} try: # sourcedata days_list = [] topic_list = Topic.objects day = pd.Period(datetime.datetime.now(),freq='D') # logger.info(type(Topic.objects)) # Topic.objects.all()Topic.objects()返回类型相同 datatype_list = Datatype_name.objects post_7days = Post.objects(Q(pt_time__gte=datetime.datetime.combine\ (date.today()-datetime.timedelta(6),datetime.time.min)) & Q(pt_time__lte=datetime.datetime.combine(date.today(), datetime.time.max))) while days_num < 7: day_dict = {} day_str = day.strftime('%Y%m%d') day_dict['time'] = day_str post = post_7days(Q(pt_time__gte=\ datetime.datetime.combine(date.today()-\ datetime.timedelta(days=days_num), \ datetime.time.min)) & \ Q(pt_time__lte=datetime.datetime.combine\ (date.today()-datetime.timedelta(days=days_num), \ datetime.time.max))) for topic in topic_list: for data in datatype_list: day_dict = {} day_dict['time'] = day_str day_dict['topic_id'] = topic._id day_dict['topic_name'] = topic.topic_name day_dict['dataTypeName'] = data.datatype_name day_dict['data_type'] = data.data_type post_datatype = post(Q(data_type=data.data_type) & Q(topic_id=topic._id)) # logger.info('post_num = ' + str(len(post_datatype))) day_dict['post_num'] = post_datatype.count() days_list.append(day_dict) # for data in datatype_list: # day_dict['dataTypeName'] = data.datatype_name # day_dict['data_type'] = data.data_type # post_datatype = post(Q(data_type=data.data_type)) # # logger.info('post_num = ' + str(len(post_datatype))) # day_dict['post_num'] = len(post_datatype) # day_dict_ = dict(day_dict) # days_list.append(day_dict_) day -= 1 days_num += 1 ####### Hot for all host posts hot_dict = {} hot_posts_temp = post_7days(Q(data_type=3)) hot_posts = hot_posts_temp.order_by \ ('-comm_num')[:10].only("_id", "url", \ "board", "title", "content", "pt_time", \ "img_url","comm_num","repost_num") hot_weibo_temp = post_7days(Q(data_type=2)) hot_weibo = hot_weibo_temp.order_by \ ('-comm_num')[:10].only("_id", "url", \ "board", "title", "content", "pt_time", \ "img_url","comm_num","repost_num") hot_poster = hot_posts_temp.order_by('-poster.post_num')[:10].only("poster") hot_weibouser_post_num = hot_weibo_temp.order_by('-poster.post_num').only("poster") hot_weibouser = [] hot_weibouser_id = [] num_ten=0 for item in hot_weibouser_post_num: if num_ten ==10: break else: if item.poster.id not in hot_weibouser_id: hot_weibouser.append(item) hot_weibouser_id.append(item.poster.id) num_ten=num_ten+1 hot_dict['hotPost'] = handle_post_list1(hot_posts) hot_dict['hotPoster'] = handle_poster_list1(hot_poster) hot_dict['hotWeibo'] = handle_post_list1(hot_weibo) # hot_dict['hotPoster'] = handle_poster_list(hot_posts) hot_dict['hotWeiboUser'] = handle_weibouser_list1(hot_weibouser) ####### map data mapData_list = [{'id':'001', 'pro':"陕西", 'nums':52 }, { 'id':'002', 'pro':"北京", 'nums':100 }, { 'id':'003', 'pro':"上海", 'nums':60 }, { 'id':'004', 'pro':"杭州", 'nums':48 }, { 'id':'005', 'pro':"南京", 'nums':50 } ] wordlist = [] wordres=Cloud_formain.objects(Q(topic_id=999)).only("word", "frequency") for worditem in wordres: temp={} temp['word']=worditem.word temp['weight']=worditem.frequency wordlist.append(temp) main_out['mapData'] = mapData_list main_out['sourceData'] = days_list main_out['Hot'] = hot_dict main_out['word_cloud'] = wordlist json_out['code'] = 0 json_out['success'] = True json_out['data'] = main_out except: traceback.print_exc() json_out['code'] = 1 json_out['data'] = {} json_out['success'] = False return HttpResponse(json.dumps(json_out, cls=MyEncoder),content_type="application/json")
def _sync_user_role_assignments(self, user_db, role_assignment_dbs, role_assignment_apis): """ Synchronize role assignments for a particular user. :param user_db: User to synchronize the assignments for. :type user_db: :class:`UserDB` :param role_assignment_dbs: Existing user role assignments. :type role_assignment_dbs: ``list`` of :class:`UserRoleAssignmentDB` :param role_assignment_apis: List of user role assignments to apply. :param role_assignment_apis: ``list`` of :class:`UserRoleAssignmentFileFormatAPI` :rtype: ``tuple`` """ db_roles = set([(entry.role, entry.source) for entry in role_assignment_dbs]) api_roles = [ list(izip_longest(entry.roles, [], fillvalue=entry.file_path)) for entry in role_assignment_apis ] api_roles = set(list(chain.from_iterable(api_roles))) # A list of new assignments which should be added to the database new_roles = api_roles.difference(db_roles) # A list of assignments which need to be updated in the database updated_roles = db_roles.intersection(api_roles) # A list of assignments which should be removed from the database removed_roles = db_roles - api_roles LOG.debug('New assignments for user "%s": %r' % (user_db.name, new_roles)) LOG.debug('Updated assignments for user "%s": %r' % (user_db.name, updated_roles)) LOG.debug('Removed assignments for user "%s": %r' % (user_db.name, removed_roles)) # Build a list of role assignments to delete roles_to_delete = updated_roles.union(removed_roles) role_assignment_dbs_to_delete = [ role_assignment_db for role_assignment_db in role_assignment_dbs if (role_assignment_db.role, role_assignment_db.source) in roles_to_delete ] for role_name, assignment_source in roles_to_delete: queryset_filter = ( Q(user=user_db.name) & Q(role=role_name) & Q(source=assignment_source) & (Q(is_remote=False) | Q(is_remote__exists=False)) ) UserRoleAssignmentDB.objects(queryset_filter).delete() LOG.debug( 'Removed role "%s" from "%s" for user "%s".' % (role_name, assignment_source, user_db.name) ) # Build a list of roles assignments to create roles_to_create = new_roles.union(updated_roles) created_role_assignment_dbs = [] for role_name, assignment_source in roles_to_create: role_db = Role.get(name=role_name) if not role_db: msg = 'Role "%s" referenced in assignment file "%s" doesn\'t exist' raise ValueError(msg % (role_name, assignment_source)) role_assignment_api = [ r for r in role_assignment_apis if r.file_path == assignment_source ][0] description = getattr(role_assignment_api, "description", None) assignment_db = rbac_service.assign_role_to_user( role_db=role_db, user_db=user_db, source=assignment_source, description=description ) created_role_assignment_dbs.append(assignment_db) LOG.debug( 'Assigned role "%s" from "%s" for user "%s".' % (role_name, assignment_source, user_db.name) ) return (created_role_assignment_dbs, role_assignment_dbs_to_delete)