def get_software_from_cpe_dic(title_this_db):
with utils.add_path(config.cpe_dic_path):
category_module = __import__(config.cpe_dic_file)
cpe_software_version_dict = category_module.cpe_software_version_dict
candidate_software_in_title_set = set()
title_word_list = title_this_db.split()
for cpe_software in cpe_software_version_dict.keys():
cpe_software_word_list = cpe_software.split()
if all(i in title_word_list for i in cpe_software_word_list):
candidate_software_in_title_set.add(cpe_software)
return candidate_software_in_title_set
def crawl_reports_by_refs(self):
dict_to_write = dict()
ref_files = os.listdir(self.cve_ref_dir)
args = []
with utils.add_path(self.cve_ref_dir):
for each_file in ref_files:
category_module = __import__(each_file.replace('.py', ''))
cve_ref_dict = category_module.cve_ref_dict
for cve_id in cve_ref_dict:
args.append((cve_id, cve_ref_dict[cve_id], dict_to_write))
self.pool.map_async(crawl_report, args)
with open(self.data_dir + 'dataset.py', 'w') as f_write:
f_write.write('version_dict = ' + str(dict_to_write))
def parse_cpe_xml():
software_version_dict = dict()
cpe_dic_name = config.cpe_dic_path + 'official-cpe-dictionary_v2.3.xml'
xmldoc = minidom.parse(cpe_dic_name)
itemlist = xmldoc.getElementsByTagName('cpe-23:cpe23-item')
# print(len(itemlist))
# print(itemlist[0].attributes['name'].value)
# print(commons.excel_data_path.replace('_a', ''))
with utils.add_path('/Users/yingdong/Desktop/vulnerability/measurement'):
module = __import__('nvd_parser')
for s in itemlist:
cpe = s.attributes['name'].value
software, version = module.get_software_name_and_version_from_cpe(cpe)
software = clean_software_name(software)
if software.startswith('a '):
print(software)
if software not in software_version_dict:
software_version_dict[software] = []
if version != '':
software_version_dict[software].append(version)
print('len(software_version_dict): ', len(software_version_dict))
write_software_name_version_dict(software_version_dict)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jun 30 15:01:08 2020
@author: suresh, eric, jessica
"""
import utils
# add path to corresponding packages
utils.add_path('/media/suresh/research/github/robotics/rl_robotics/packages/relay_policy_learning/adept_envs')
utils.add_path('/media/suresh/research/github/robotics/rl_robotics/packages/puppet/vive/source')
utils.add_path('/media/suresh/research/github/robotics/rl_robotics/packages/mjrl')
import os
import pickle
import numpy as np
from parse_mjl import parse_mjl_logs, viz_parsed_mjl_logs
from mjrl.utils.gym_env import GymEnv
import adept_envs
import gym
import cv2
# playback demos and get data(physics respected)
def gather_training_data(env, data, pkl_seq, save_path, width = 300, height = 300, render=None):
env = env.env
# initialize
env.reset()
def copy2local(script_name):
add_path(dirname(__file__))
run_copy_to_local(script_name)
def test():
add_path(dirname(__file__))
print(run_once('test_script', 'test_func'))
def __createSingleImageDataset(self, img, style):
with add_path(self.pix2pixDir):
dataset = SingleItemDataset(self.opts, img, style)
return dataset
import os
from utils import add_path
import numpy as np
from skimage import morphology
from PIL import Image
from algorithms.image_padding import pad_image, unpad_image
from pipeline.render_skeleton import blur_skeleton
import gc
import argparse
with add_path(
os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'ext',
'pix2pix')):
from options.test_options import TestOptions
from models import create_model
from data.single_item_dataset import SingleItemDataset
from util.util import tensor2im
class PenStyleTransfer:
def __init__(self):
self.dir_path = os.path.dirname(os.path.realpath(__file__))
self.pix2pixDir = os.path.join(self.dir_path, '..', 'ext', 'pix2pix')
netName = 'cond_pix2pix_2048_asymmetric'
self.opts = TestOptions().parse([
'--dataroot', '', '--model', 'cond_pix2pix', '--checkpoints_dir',
import pickle
from model import NLG
from data_engine import DataEngine
from text_token import _UNK, _PAD, _BOS, _EOS
import torch
import torch.nn as nn
import numpy as np
import os
from utils import print_config, add_path
from model_utils import get_embeddings
from argument import define_arguments
from utils import get_time
_, args = define_arguments()
args = add_path(args)
if args.verbose_level > 0:
print_config(args)
use_cuda = torch.cuda.is_available()
train_data_engine = DataEngine(
data_dir=args.data_dir,
dataset=args.dataset,
save_path=args.train_data_file,
vocab_path=args.vocab_file,
is_spacy=args.is_spacy,
is_lemma=args.is_lemma,
fold_attr=args.fold_attr,
use_punct=args.use_punct,
vocab_size=args.vocab_size,
n_layers=args.n_layers,
current_version=CURRENT_VERSION)
if latest_version:
print(
'DDRecorder有更新，版本号: {} 请尽快到https://github.com/AsaChiri/DDRecorder/releases 下载最新版'
.format(str(latest_version)))
else:
print('DDRecorder已是最新版本！')
if __name__ == "__main__":
freeze_support()
vt = versionThread()
vt.start()
if utils.is_windows():
utils.add_path("./ffmpeg/bin")
try:
if len(sys.argv) > 1:
all_config_filename = sys.argv[1]
with open(all_config_filename, "r", encoding="UTF-8") as f:
all_config = json.load(f)
else:
with open("config.json", "r", encoding="UTF-8") as f:
all_config = json.load(f)
except Exception as e:
print("解析配置文件时出现错误，请检查配置文件！")
print("错误详情：" + str(e))
os.system('pause')
utils.check_and_create_dir(
def read_id2word():
with utils.add_path(config.labeled_re_data_write_path):
category_module = __import__('id2word_file')
id2word = category_module.id2word
return id2word
def read_char2id():
with utils.add_path(config.labeled_re_data_write_path):
category_module = __import__('char2id_file')
char2id = category_module.char2id
return char2id
import logging
import os
import gc
import numpy as np
from utils import add_path
from datastructures.PenPosition import PenPosition
with add_path(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'ext', 'graves')):
import drawing
from rnn import rnn
def preprocess(sampleStrokes):
strokes = sampleStrokes
coords = []
if strokes[0].penUp < 0.5:
coords = [[0, 0, 1]]
for i, point in enumerate(strokes):
coords.append([
int(point.pos[0]),
-1*int(point.pos[1]),
point.penUp
])
coords = np.array(coords)
import copy
import os
import sys
import utils
from api_connector import ApiConnector
from article_reader import ArticleReader
from batch_processor import BatchProcessor
from data_saver import ArticleSaver, TitleSaver, TextSaver
from article_filter import ArticleFilter
args = utils.parse_args()
cfg = utils.read_config('config.json')
utils.add_path(args['dir'], cfg['files'])
afilter = ArticleFilter(**cfg['filter'])
# Read current level: relevant only when not seeding
try:
curr_level = int(TextSaver.read_file(cfg['files']['curr_level']))
except FileNotFoundError:
curr_level = 1
if curr_level > args['levels']:
sys.exit("ERR: Already crawled all permitted levels. Quitting...")
# Get lists stored in files from previous crawls
all_discards = TitleSaver.read_title_file(cfg['files']['discarded'])
all_redirects = TitleSaver.read_title_file(cfg['files']['redirected'])
all_titles = TitleSaver.read_title_file(cfg['files']['crawled'])
all_ids = TitleSaver.read_title_file(cfg['files']['crawled_ids'])
