def migrate(input_cfg, output_cfg): d = utils.read_conf(input_cfg) val = {} for k in keys: val[k] = d.get(k, '') if val['db_host'] == 'postgresql' and val['db_port'] == '5432': val['external_db'] = False else: val['external_db'] = True # If using default filesystem, didn't need registry_storage_provider_config config if val['registry_storage_provider_name'] == 'filesystem' and not val.get( 'registry_storage_provider_config'): val['storage_provider_info'] = '' else: val['storage_provider_info'] = utils.get_storage_provider_info( val['registry_storage_provider_name'], val['registry_storage_provider_config']) if val['redis_host'] == 'redis' and val['redis_port'] == '6379': val['external_redis'] = False else: val['registry_db_index'], val['jobservice_db_index'], val[ 'chartmuseum_db_index'] = map(int, val['redis_db_index'].split(',')) val['external_redis'] = True this_dir = os.path.dirname(__file__) tpl = Environment( loader=FileSystemLoader(this_dir)).get_template('harbor.yml.jinja') with open(output_cfg, 'w') as f: f.write(tpl.render(**val))
def migrate(input_cfg, output_cfg): d = utils.read_conf(input_cfg) val = {} for k in keys: val[k] = d.get(k, '') tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.yml.tpl') utils.render(tpl_path, output_cfg, **val)
def init_start(): #read global config file splt_ch = '|' param = {'school_name': '', 'pre_reserve_time': ''} # get comandline param if len(sys.argv)-1 != len(param.keys()): raise Exception('#param num do not equals to len(param.keys())') param['school_name'] = sys.argv[1] param['pre_reserve_time'] = sys.argv[2] # read today_task list file_name = 'clssrm_id_and_today_task.conf' file_path = './classroom' + '/' + param['school_name'] + '/' +file_name section = 'today_task' task_ls = utils.read_conf(file_path, section=section).keys() # random or in order random_model = False if random_model == True: random.shuffle(task_ls) else: # sort task_ls.sorted(key=lambda t: int(t.split(splt_ch, 1)[0]), reverse=False) debug_p(task_ls) for task in task_ls: task_param_d = utils.parse_command(task, type='simplify')
def run(run_once=True): #TODO: turn this into a daemon conf = utils.read_conf("config.toml") if run_once: run_steps(conf) else: while True: poll_frequency = int(conf["global"]["poll_frequency"]) * 60 run_steps(conf) sleep(poll_frequency)
def info(self): """Called by ABCI when the app first starts.""" self.conf = utils.read_conf() self.db = utils.DatabaseProvider(conf=self.conf) r = ResponseInfo() r.last_block_height = self.db.get_block_height() r.last_block_app_hash = self.db.get_block_app_hash().encode() return r
def migrate(input_cfg, output_cfg): config_dict = utils.read_conf(input_cfg) current_dir = os.path.dirname(__file__) tpl = Environment(loader=FileSystemLoader(current_dir), undefined=StrictUndefined, trim_blocks=True, lstrip_blocks=True).get_template('harbor.yml.jinja') with open(output_cfg, 'w') as f: f.write(tpl.render(**config_dict))
def migrate(input_cfg, output_cfg): d = utils.read_conf(input_cfg) val = {} for k in keys: val[k] = d.get(k, '') #append registry to no_proxy np_list = d.get('no_proxy', '').split(',') new_np_list = ['core' if x == 'ui' else x for x in np_list] val['no_proxy'] = ','.join(new_np_list) tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.cfg.tpl') utils.render(tpl_path, output_cfg, **val)
def run(run_once=True): #TODO: turn this into a daemon conf = utils.read_conf("config.toml") if run_once: run_steps(conf) else: while True: poll_frequency = int(conf["global"]["poll_frequency"]) * 60 run_steps(conf) logger.info("all jobs done... sleeping {%s} seconds", poll_frequency) sleep(poll_frequency)
def main(): """ 监控器方法 """ # 判断参数列表长度,看是否提供了配置文件目录 argv_list = sys.argv if len(argv_list) != 2: # 如果参数列表长度不是2,表明给的参数不止一个或没给参数,报错提示并退出 raise '参数长度不正确,请给出仅一个配置文件路径参数,如果路径含有空格请使用单/双引号包裹' # 读取配置文件 conf = utils.read_conf(argv_list[1]) # 遍历配置中要管理的目录 # 将需要进行备份的日志文件进行备份操作 # 并且将它们的路径记录到列表中,以备进行邮件提醒 backuped_logs_list = [] for path in conf['logs_path']: # 获取当前path下日志文件路径及大小关系映射字典 logs_path_size_dict = utils.get_logs_path_size(path, conf['logs_suffix']) # 获取要进行备份的日志路径们 # (先进行配置文件中阈值数值单位转换) backup_size_byte = utils.covert_unit_of_filesize( conf['backup_size'], conf['unit'], 'B') to_backup_logs_list = utils.get_to_backup_logs(logs_path_size_dict, backup_size_byte) # 遍历需要备份的日志文件,进行备份 for to_backup_log in to_backup_logs_list: utils.backup_conf(to_backup_log) # 记录到backuped_logs_list里 backuped_logs_list.append(to_backup_log) # 判断是否需要email提醒 # 本次没有进行备份操作的话也不发送email if conf['need_email'] and backuped_logs_list: # 创建邮件内容 message = ''' 你的{}服务器内,以下日志文件大小超过设置阈值,已进行备份: {} '''.format(conf['server_name'], backuped_logs_list) # 发送电子邮件 utils.send_email( from_email=conf['from_email'], email_password=conf['email_password'], to_email=conf['to_email'], message=message, )
def migrate(input_cfg, output_cfg): d = utils.read_conf(input_cfg) keys = list(default.keys()) keys.extend([ 'hostname', 'ui_url_protocol', 'max_job_workers', 'customize_crt', 'ssl_cert', 'ssl_cert_key', 'secretkey_path', 'admiral_url', 'db_password', 'clair_db_password' ]) val = {} for k in keys: if k in d: val[k] = d[k] else: val[k] = default[k] tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.cfg.tpl') utils.render(tpl_path, output_cfg, **val)
def migrate(input_cfg, output_cfg): d = utils.read_conf(input_cfg) val = {} for k in keys: val[k] = d.get(k, '') #append registry to no_proxy np_list = d.get('no_proxy', '').split(',') if not 'registry' in np_list: np_list.append('registry') val['no_proxy'] = ','.join(np_list) #handle harbor db information, if it previously pointed to internal mariadb, point it to the new default db instance of pgsql, #update user to default pgsql user. if 'mysql' == d['db_host']: val['db_host'] = 'postgresql' val['db_port'] = '5432' val['db_user'] = '******' #handle clair db information, if it pointed to internal pgsql in previous deployment, point it to the new default db instance of pgsql, #the user should be the same user as harbor db if 'postgres' == d['clair_db_host']: val['clair_db_host'] = 'postgresql' val['cliar_db_user'] = val['db_user'] val['clair_db_password'] = val['db_password'] tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.cfg.tpl') utils.render(tpl_path, output_cfg, **val)
def migrate(input_cfg, output_cfg): d = utils.read_conf(input_cfg) val = {} for k in keys: val[k] = d.get(k,'') #append registry to no_proxy np_list = d.get('no_proxy','').split(',') if not 'registry' in np_list: np_list.append('registry') val['no_proxy'] = ','.join(np_list) #handle harbor db information, if it previously pointed to internal mariadb, point it to the new default db instance of pgsql, #update user to default pgsql user. if 'mysql' == d['db_host']: val['db_host'] = 'postgresql' val['db_port'] = '5432' val['db_user'] = '******' #handle clair db information, if it pointed to internal pgsql in previous deployment, point it to the new default db instance of pgsql, #the user should be the same user as harbor db if 'postgres' == d['clair_db_host']: val['clair_db_host'] = 'postgresql' val['cliar_db_user'] = val['db_user'] val['clair_db_password'] = val['db_password'] tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.cfg.tpl') utils.render(tpl_path, output_cfg, **val)
def main(args): args = read_conf(args.cfg, args) torch.manual_seed(args.seed) np.random.seed(args.seed) if args.dataset == 'timit': train_dataset = TIMIT(data_root=args.data_root, datalist_root=args.datalist_root, train=True, oversampling=args.oversampling) test_dataset = TIMIT(data_root=args.data_root, datalist_root=args.datalist_root, train=False) elif args.dataset == 'libri': raise NotImplementedError else: raise NotImplementedError cost = nn.NLLLoss() CNN_arch = { 'input_dim': train_dataset.wlen, 'fs': args.fs, 'cnn_N_filt': args.cnn_N_filt, 'cnn_len_filt': args.cnn_len_filt, 'cnn_max_pool_len': args.cnn_max_pool_len, 'cnn_use_laynorm_inp': args.cnn_use_laynorm_inp, 'cnn_use_batchnorm_inp': args.cnn_use_batchnorm_inp, 'cnn_use_laynorm': args.cnn_use_laynorm, 'cnn_use_batchnorm': args.cnn_use_batchnorm, 'cnn_act': args.cnn_act, 'cnn_drop': args.cnn_drop, } DNN1_arch = { 'fc_lay': args.fc_lay, 'fc_drop': args.fc_drop, 'fc_use_batchnorm': args.fc_use_batchnorm, 'fc_use_laynorm': args.fc_use_laynorm, 'fc_use_laynorm_inp': args.fc_use_laynorm_inp, 'fc_use_batchnorm_inp': args.fc_use_batchnorm_inp, 'fc_act': args.fc_act, } DNN2_arch = { 'input_dim': args.fc_lay[-1], 'fc_lay': args.class_lay, 'fc_drop': args.class_drop, 'fc_use_batchnorm': args.class_use_batchnorm, 'fc_use_laynorm': args.class_use_laynorm, 'fc_use_laynorm_inp': args.class_use_laynorm_inp, 'fc_use_batchnorm_inp': args.class_use_batchnorm_inp, 'fc_act': args.class_act, } model = SpeakerIDNet(CNN_arch, DNN1_arch, DNN2_arch) if args.pt_file != '': print("load model from:", args.pt_file) checkpoint_load = torch.load(args.pt_file) ext = os.path.splitext(args.pt_file)[1] if ext == '.pkl': model.load_raw_state_dict(checkpoint_load) elif ext == '.pickle': model.load_state_dict(checkpoint_load) elif ext == '.pth': load_checkpoint(model, args.pt_file) else: raise NotImplementedError model = model.cuda() if args.eval: print('only eval the model') evaluate(model, test_dataset, cost) return else: print("train the model") optimizer = optim.RMSprop(model.parameters(), lr=args.lr, alpha=0.95, eps=1e-8) train_dataloader = DataLoader(train_dataset, args.batch_size, shuffle=True, num_workers=8, pin_memory=True) test_dataloader = DataLoader(test_dataset, 1, shuffle=False, num_workers=8, pin_memory=True) trainer = ClassifierTrainer(model, train_dataloader, optimizer, cost, batch_process, args.output_dir, 0, test_dataloader, eval_every=args.N_eval_epoch, print_every=args.print_every) trainer.run(args.N_epochs)
### test (20210505_235209) in_file = "data/wikitestko_from_table_not_h_4.jsonl" out_file = f"output/test_out_ko_from_table_not_h_4_beam-{args.beam_size}_top-{args.topk}.jsonl" label_file = "WikiSQL/data/test.jsonl" db_file = "WikiSQL/data/test.db" model_out_file = f"output/test_model_out_ko_from_table_not_h_4_beam-{args.beam_size}_top-{args.topk}.pkl" ###================================================================================================### # All Best model_path = "output/20210505_235209" epoch = 4 engine = DBEngine(db_file) config = utils.read_conf(os.path.join(model_path, "model.conf")) # config["DEBUG"] = 1 featurizer = HydraFeaturizer(config) pred_data = SQLDataset(in_file, config, featurizer, False) print("num of samples: {0}".format(len(pred_data.input_features))) ##======================EG + TOP_k=============================## model = create_model(config, is_train=False) model.load(model_path, epoch) if "DEBUG" in config: model_out_file = model_out_file + ".partial" if os.path.exists(model_out_file): model_outputs = pickle.load(open(model_out_file, "rb"))
print(eval_file + ": " + result_str) if "DEBUG" in self.config: for text in sq: print(text[0] + ":" + text[1] + "\t" + text[2]) else: with open(self.eval_history_file, "a+", encoding="utf8") as f: f.write("[{0}, epoch {1}] ".format(eval_file, epochs) + result_str + "\n") bad_case_file = os.path.join( self.bad_case_dir, "{0}_epoch_{1}.log".format(eval_file, epochs)) with open(bad_case_file, "w", encoding="utf8") as f: for text in sq: f.write(text[0] + ":" + text[1] + "\t" + text[2] + "\n") if __name__ == "__main__": os.environ["CUDA_VISIBLE_DEVICES"] = "3" config = utils.read_conf(os.path.join("conf", "wikisql.conf")) config["DEBUG"] = 1 config["num_train_steps"] = 1000 config["num_warmup_steps"] = 100 featurizer = HydraFeaturizer(config) model = create_model(config, is_train=True, num_gpu=1) evaluator = HydraEvaluator("output", config, featurizer, model, "debug evaluator") evaluator.eval(0)
"--model_path", help="trained model folder path (used in eval, predict and export mode)") parser.add_argument( "--epoch", help="epochs to restore (used in eval, predict and export mode)") parser.add_argument("--gpu", type=str, default=None, help="gpu id") parser.add_argument("--note", type=str) args = parser.parse_args() if args.job == "train": if args.gpu is not None: os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu # os.environ['TF_ENABLE_AUTO_MIXED_PRECISION'] = '1' conf_path = os.path.abspath(args.conf) config = utils.read_conf(conf_path) note = args.note if args.note else "" script_path = os.path.dirname(os.path.abspath(sys.argv[0])) output_path = args.output_path model_name = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") model_path = os.path.join(output_path, model_name) if "DEBUG" not in config: if not os.path.exists(output_path): os.mkdir(output_path) if not os.path.exists(model_path): os.mkdir(model_path) shutil.copyfile(conf_path, os.path.join(model_path, "model.conf"))
def execute_one_test(dataset, shot, model_moment, epoch): os.environ["CUDA_VISIBLE_DEVICES"] = "0, 1, 2, 3" model_path = "output/" + model_moment in_file = "data/wiki{}_content.jsonl".format( dataset) if shot == "orig" else "data/wiki{}_{}_content.jsonl".format( dataset, shot) db_file = "WikiSQL/data/{}.db".format(dataset) label_file = "WikiSQL/data/{}.jsonl".format( dataset) if shot == "orig" else "WikiSQL/data_{}/{}.jsonl".format( shot, dataset) out_path = "predictions/{}_{}_{}_{}".format(model_moment, epoch, dataset, shot) if not os.path.exists(out_path): os.mkdir(out_path) out_file = os.path.join(out_path, "out.jsonl") eg_out_file = os.path.join(out_path, "out_eg.jsonl") model_out_file = os.path.join(out_path, "model_out.pkl") test_result_file = os.path.join(out_path, "result.txt") engine = DBEngine(db_file) config = utils.read_conf(os.path.join(model_path, "model.conf")) # config["DEBUG"] = 1 featurizer = HydraFeaturizer(config) pred_data = SQLDataset(in_file, config, featurizer, False) print("num of samples: {0}".format(len(pred_data.input_features))) model = create_model(config, is_train=False) model.load(model_path, epoch) if "DEBUG" in config: model_out_file = model_out_file + ".partial" model_outputs = model.dataset_inference(pred_data) pickle.dump(model_outputs, open(model_out_file, "wb")) # model_outputs = pickle.load(open(model_out_file, "rb")) print("===HydraNet===") pred_sqls = model.predict_SQL(pred_data, model_outputs=model_outputs) with open(out_file, "w") as g: for pred_sql in pred_sqls: # print(pred_sql) result = {"query": {}} result["query"]["agg"] = int(pred_sql[0]) result["query"]["sel"] = int(pred_sql[1]) result["query"]["conds"] = [(int(cond[0]), int(cond[1]), str(cond[2])) for cond in pred_sql[2]] g.write(json.dumps(result) + "\n") normal_res = print_metric(label_file, out_file, db_file) print("===HydraNet+EG===") pred_sqls = model.predict_SQL_with_EG(engine, pred_data, model_outputs=model_outputs) with open(eg_out_file, "w") as g: for pred_sql in pred_sqls: # print(pred_sql) result = {"query": {}} result["query"]["agg"] = int(pred_sql[0]) result["query"]["sel"] = int(pred_sql[1]) result["query"]["conds"] = [(int(cond[0]), int(cond[1]), str(cond[2])) for cond in pred_sql[2]] g.write(json.dumps(result) + "\n") eg_res = print_metric(label_file, eg_out_file, db_file) with open(test_result_file, "w") as g: g.write("normal results:\n" + normal_res + "eg results:\n" + eg_res)
def main(args): speaker_cfg = args.speaker_cfg speech_cfg = args.speech_cfg args_speaker = read_conf(speaker_cfg, deepcopy(args)) args_speaker.model_path = args.speaker_model args_speech = read_conf(speech_cfg, deepcopy(args)) args_speech.model_path = args.speech_model torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False print("set seed: ", args_speaker.optimization.seed) torch.manual_seed(args_speaker.optimization.seed) np.random.seed(args_speaker.optimization.seed) random.seed(args_speaker.optimization.seed) torch.cuda.set_device(args.local_rank) if not args.no_dist: torch.distributed.init_process_group(backend="nccl") train_dataset = TIMIT_speaker(args.data_root, train=True, phoneme=True, norm_factor=True) test_dataset = TIMIT_speaker(args.data_root, train=False, phoneme=True, norm_factor=True) pretrained_models = get_pretrained_models(args_speaker, args_speech) loss_factors = { "speaker": args.speaker_factor, "speech": args.speech_factor, "norm": args.norm_factor } if args.target < 0: # non-targeted speaker_loss = SpeakerLoss(pretrained_models['speaker']) else: # targeted attack speaker_loss = SpeakerLossTarget(pretrained_models['speaker'], args.target) loss_all = {} loss_all['speech'] = { 'model': pretrained_models['speech'], 'factor': loss_factors['speech'], 'loss_func': SpeechLoss(pretrained_models['speech'], factor_kld=args.speech_kld_factor) } loss_all['speaker'] = { 'model': pretrained_models['speaker'], 'factor': loss_factors['speaker'], 'loss_func': speaker_loss } loss_all['norm'] = { 'loss_func': MSEWithThreshold(args.norm_clip), 'factor': loss_factors['norm'] } cost = AdversarialLoss(loss_all) model = SpeechTransformer(args.channel, args.kernel_size, args.dilation, args.sample, args.noise_scale) if args.pt_file != 'none': print("load model from:", args.pt_file) if os.path.splitext(args.pt_file)[1] == '.pkl': checkpoint_load = torch.load(args.pt_file) model.load_raw_state_dict(checkpoint_load) else: load_checkpoint(model, args.pt_file) model = model.cuda() if args.eval: assert args.pt_file != 'none', "no pretrained model is provided!" print('only eval the model') evaluate(model, test_dataset, cost) return if args.test: assert args.pt_file != 'none', "no pretrained model is provided!" print("only test the model") filename_list = open("./data/TIMIT/speaker/test.scp", 'r').readlines() filename_list = [_f.strip() for _f in filename_list] label_dict = np.load( os.path.join(args.data_root, "processed", "TIMIT_labels.npy")).item() test_wav(model, filename_list, args.data_root, os.path.join(args.data_root, "output"), pretrained_models['speaker'], label_dict, args.target) return if args.cpu_test: assert args.pt_file != 'none', "no pretrained model is provided!" print("only cpu test the model") filename_list = open("./data/TIMIT/speaker/test.scp", 'r').readlines() filename_list = [_f.strip() for _f in filename_list] label_dict = np.load( os.path.join(args.data_root, "processed", "TIMIT_labels.npy")).item() test_wav_cpu(model, filename_list, args.data_root, os.path.join(args.data_root, "output"), pretrained_models['speaker'], label_dict, args.target) return print("train the model") batch_process = batch_process_speaker eval_hook = EvalHook() optimizer = optim.Adam(model.parameters(), lr=args_speaker.optimization.lr, betas=(0.95, 0.999)) lr_scheduler = optim.lr_scheduler.StepLR(optimizer, 2, 0.5) if args.no_dist: kwarg = { 'shuffle': True, 'worker_init_fn': partial(_init_fn, seed=args_speaker.optimization.seed) } else: train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset) kwarg = { 'sampler': train_sampler, 'worker_init_fn': partial(_init_fn, seed=args_speaker.optimization.seed) } train_dataloader = DataLoader(train_dataset, args_speaker.optimization.batch_size, num_workers=args.num_workers, pin_memory=True, **kwarg) test_dataloader = DataLoader(test_dataset, args_speaker.optimization.batch_size, shuffle=False, num_workers=args.num_workers, pin_memory=True) trainer = ClassifierTrainer( model, train_dataloader, optimizer, cost, batch_process, args.output_dir, 0, test_dataloader, eval_hook=eval_hook, eval_every=args_speaker.optimization.N_eval_epoch, print_every=args_speaker.optimization.print_every, lr_scheduler=lr_scheduler) trainer.logger.info(args) trainer.run(args_speaker.optimization.N_epochs)
import tweepy def inputs(): keyword = raw_input("Enter keyword to test: ") return keyword #def checkFilter(keywords, lines): # while keywords in lines: if __name__=="__main__": try: scrape() cfs = ChiFeatureSelector('trending.%d.json'%os.getpid(), 'nontrending.%d.ujson'%os.getpid()) except: classify = classifier.HashtagClassifier() classify.condProb = utils.read_conf('classifierTrained.json') classify.prior = utils.read_conf('classifier_prior.json') while True: keyword = re.sub("""[\s/:*"<>?|\\.;'\[\]]+""", '', inputs()) if not keyword: print 'Please enter a valid phrase' continue try: scrapeTrends.search_tweet(keyword) except tweepy.TweepError: print 'Please enter a valid phrase' continue try: tweets = utils.read_tweets('tweets/tweets.%(name)s.json'%{'name':keyword}) except: