def get_software_from_cpe_dic(title_this_db): with utils.add_path(config.cpe_dic_path): category_module = __import__(config.cpe_dic_file) cpe_software_version_dict = category_module.cpe_software_version_dict candidate_software_in_title_set = set() title_word_list = title_this_db.split() for cpe_software in cpe_software_version_dict.keys(): cpe_software_word_list = cpe_software.split() if all(i in title_word_list for i in cpe_software_word_list): candidate_software_in_title_set.add(cpe_software) return candidate_software_in_title_set
def crawl_reports_by_refs(self): dict_to_write = dict() ref_files = os.listdir(self.cve_ref_dir) args = [] with utils.add_path(self.cve_ref_dir): for each_file in ref_files: category_module = __import__(each_file.replace('.py', '')) cve_ref_dict = category_module.cve_ref_dict for cve_id in cve_ref_dict: args.append((cve_id, cve_ref_dict[cve_id], dict_to_write)) self.pool.map_async(crawl_report, args) with open(self.data_dir + 'dataset.py', 'w') as f_write: f_write.write('version_dict = ' + str(dict_to_write))
def parse_cpe_xml(): software_version_dict = dict() cpe_dic_name = config.cpe_dic_path + 'official-cpe-dictionary_v2.3.xml' xmldoc = minidom.parse(cpe_dic_name) itemlist = xmldoc.getElementsByTagName('cpe-23:cpe23-item') # print(len(itemlist)) # print(itemlist[0].attributes['name'].value) # print(commons.excel_data_path.replace('_a', '')) with utils.add_path('/Users/yingdong/Desktop/vulnerability/measurement'): module = __import__('nvd_parser') for s in itemlist: cpe = s.attributes['name'].value software, version = module.get_software_name_and_version_from_cpe(cpe) software = clean_software_name(software) if software.startswith('a '): print(software) if software not in software_version_dict: software_version_dict[software] = [] if version != '': software_version_dict[software].append(version) print('len(software_version_dict): ', len(software_version_dict)) write_software_name_version_dict(software_version_dict)
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Tue Jun 30 15:01:08 2020 @author: suresh, eric, jessica """ import utils # add path to corresponding packages utils.add_path('/media/suresh/research/github/robotics/rl_robotics/packages/relay_policy_learning/adept_envs') utils.add_path('/media/suresh/research/github/robotics/rl_robotics/packages/puppet/vive/source') utils.add_path('/media/suresh/research/github/robotics/rl_robotics/packages/mjrl') import os import pickle import numpy as np from parse_mjl import parse_mjl_logs, viz_parsed_mjl_logs from mjrl.utils.gym_env import GymEnv import adept_envs import gym import cv2 # playback demos and get data(physics respected) def gather_training_data(env, data, pkl_seq, save_path, width = 300, height = 300, render=None): env = env.env # initialize env.reset()
def copy2local(script_name): add_path(dirname(__file__)) run_copy_to_local(script_name)
def test(): add_path(dirname(__file__)) print(run_once('test_script', 'test_func'))
def __createSingleImageDataset(self, img, style): with add_path(self.pix2pixDir): dataset = SingleItemDataset(self.opts, img, style) return dataset
import os from utils import add_path import numpy as np from skimage import morphology from PIL import Image from algorithms.image_padding import pad_image, unpad_image from pipeline.render_skeleton import blur_skeleton import gc import argparse with add_path( os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'ext', 'pix2pix')): from options.test_options import TestOptions from models import create_model from data.single_item_dataset import SingleItemDataset from util.util import tensor2im class PenStyleTransfer: def __init__(self): self.dir_path = os.path.dirname(os.path.realpath(__file__)) self.pix2pixDir = os.path.join(self.dir_path, '..', 'ext', 'pix2pix') netName = 'cond_pix2pix_2048_asymmetric' self.opts = TestOptions().parse([ '--dataroot', '', '--model', 'cond_pix2pix', '--checkpoints_dir',
import pickle from model import NLG from data_engine import DataEngine from text_token import _UNK, _PAD, _BOS, _EOS import torch import torch.nn as nn import numpy as np import os from utils import print_config, add_path from model_utils import get_embeddings from argument import define_arguments from utils import get_time _, args = define_arguments() args = add_path(args) if args.verbose_level > 0: print_config(args) use_cuda = torch.cuda.is_available() train_data_engine = DataEngine( data_dir=args.data_dir, dataset=args.dataset, save_path=args.train_data_file, vocab_path=args.vocab_file, is_spacy=args.is_spacy, is_lemma=args.is_lemma, fold_attr=args.fold_attr, use_punct=args.use_punct, vocab_size=args.vocab_size, n_layers=args.n_layers,
current_version=CURRENT_VERSION) if latest_version: print( 'DDRecorder有更新,版本号: {} 请尽快到https://github.com/AsaChiri/DDRecorder/releases 下载最新版' .format(str(latest_version))) else: print('DDRecorder已是最新版本!') if __name__ == "__main__": freeze_support() vt = versionThread() vt.start() if utils.is_windows(): utils.add_path("./ffmpeg/bin") try: if len(sys.argv) > 1: all_config_filename = sys.argv[1] with open(all_config_filename, "r", encoding="UTF-8") as f: all_config = json.load(f) else: with open("config.json", "r", encoding="UTF-8") as f: all_config = json.load(f) except Exception as e: print("解析配置文件时出现错误,请检查配置文件!") print("错误详情:" + str(e)) os.system('pause') utils.check_and_create_dir(
def read_id2word(): with utils.add_path(config.labeled_re_data_write_path): category_module = __import__('id2word_file') id2word = category_module.id2word return id2word
def read_char2id(): with utils.add_path(config.labeled_re_data_write_path): category_module = __import__('char2id_file') char2id = category_module.char2id return char2id
import logging import os import gc import numpy as np from utils import add_path from datastructures.PenPosition import PenPosition with add_path(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'ext', 'graves')): import drawing from rnn import rnn def preprocess(sampleStrokes): strokes = sampleStrokes coords = [] if strokes[0].penUp < 0.5: coords = [[0, 0, 1]] for i, point in enumerate(strokes): coords.append([ int(point.pos[0]), -1*int(point.pos[1]), point.penUp ]) coords = np.array(coords)
import copy import os import sys import utils from api_connector import ApiConnector from article_reader import ArticleReader from batch_processor import BatchProcessor from data_saver import ArticleSaver, TitleSaver, TextSaver from article_filter import ArticleFilter args = utils.parse_args() cfg = utils.read_config('config.json') utils.add_path(args['dir'], cfg['files']) afilter = ArticleFilter(**cfg['filter']) # Read current level: relevant only when not seeding try: curr_level = int(TextSaver.read_file(cfg['files']['curr_level'])) except FileNotFoundError: curr_level = 1 if curr_level > args['levels']: sys.exit("ERR: Already crawled all permitted levels. Quitting...") # Get lists stored in files from previous crawls all_discards = TitleSaver.read_title_file(cfg['files']['discarded']) all_redirects = TitleSaver.read_title_file(cfg['files']['redirected']) all_titles = TitleSaver.read_title_file(cfg['files']['crawled']) all_ids = TitleSaver.read_title_file(cfg['files']['crawled_ids'])