def migrate(input_cfg, output_cfg):
d = utils.read_conf(input_cfg)
val = {}
for k in keys:
val[k] = d.get(k, '')
if val['db_host'] == 'postgresql' and val['db_port'] == '5432':
val['external_db'] = False
else:
val['external_db'] = True
# If using default filesystem, didn't need registry_storage_provider_config config
if val['registry_storage_provider_name'] == 'filesystem' and not val.get(
'registry_storage_provider_config'):
val['storage_provider_info'] = ''
else:
val['storage_provider_info'] = utils.get_storage_provider_info(
val['registry_storage_provider_name'],
val['registry_storage_provider_config'])
if val['redis_host'] == 'redis' and val['redis_port'] == '6379':
val['external_redis'] = False
else:
val['registry_db_index'], val['jobservice_db_index'], val[
'chartmuseum_db_index'] = map(int,
val['redis_db_index'].split(','))
val['external_redis'] = True
this_dir = os.path.dirname(__file__)
tpl = Environment(
loader=FileSystemLoader(this_dir)).get_template('harbor.yml.jinja')
with open(output_cfg, 'w') as f:
f.write(tpl.render(**val))
def migrate(input_cfg, output_cfg):
d = utils.read_conf(input_cfg)
val = {}
for k in keys:
val[k] = d.get(k, '')
tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.yml.tpl')
utils.render(tpl_path, output_cfg, **val)
def init_start():
#read global config file
splt_ch = '|'
param = {'school_name': '',
'pre_reserve_time': ''}
# get comandline param
if len(sys.argv)-1 != len(param.keys()):
raise Exception('#param num do not equals to len(param.keys())')
param['school_name'] = sys.argv[1]
param['pre_reserve_time'] = sys.argv[2]
# read today_task list
file_name = 'clssrm_id_and_today_task.conf'
file_path = './classroom' + '/' + param['school_name'] + '/' +file_name
section = 'today_task'
task_ls = utils.read_conf(file_path, section=section).keys()
# random or in order
random_model = False
if random_model == True:
random.shuffle(task_ls)
else:
# sort
task_ls.sorted(key=lambda t: int(t.split(splt_ch, 1)[0]), reverse=False)
debug_p(task_ls)
for task in task_ls:
task_param_d = utils.parse_command(task, type='simplify')
def run(run_once=True):
#TODO: turn this into a daemon
conf = utils.read_conf("config.toml")
if run_once:
run_steps(conf)
else:
while True:
poll_frequency = int(conf["global"]["poll_frequency"]) * 60
run_steps(conf)
sleep(poll_frequency)
def info(self):
"""Called by ABCI when the app first starts."""
self.conf = utils.read_conf()
self.db = utils.DatabaseProvider(conf=self.conf)
r = ResponseInfo()
r.last_block_height = self.db.get_block_height()
r.last_block_app_hash = self.db.get_block_app_hash().encode()
return r
def migrate(input_cfg, output_cfg):
config_dict = utils.read_conf(input_cfg)
current_dir = os.path.dirname(__file__)
tpl = Environment(loader=FileSystemLoader(current_dir),
undefined=StrictUndefined,
trim_blocks=True,
lstrip_blocks=True).get_template('harbor.yml.jinja')
with open(output_cfg, 'w') as f:
f.write(tpl.render(**config_dict))
def migrate(input_cfg, output_cfg):
d = utils.read_conf(input_cfg)
val = {}
for k in keys:
val[k] = d.get(k, '')
#append registry to no_proxy
np_list = d.get('no_proxy', '').split(',')
new_np_list = ['core' if x == 'ui' else x for x in np_list]
val['no_proxy'] = ','.join(new_np_list)
tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.cfg.tpl')
utils.render(tpl_path, output_cfg, **val)
def run(run_once=True):
#TODO: turn this into a daemon
conf = utils.read_conf("config.toml")
if run_once:
run_steps(conf)
else:
while True:
poll_frequency = int(conf["global"]["poll_frequency"]) * 60
run_steps(conf)
logger.info("all jobs done... sleeping {%s} seconds",
poll_frequency)
sleep(poll_frequency)
def main():
"""
监控器方法
"""
# 判断参数列表长度，看是否提供了配置文件目录
argv_list = sys.argv
if len(argv_list) != 2:
# 如果参数列表长度不是2，表明给的参数不止一个或没给参数，报错提示并退出
raise '参数长度不正确，请给出仅一个配置文件路径参数，如果路径含有空格请使用单/双引号包裹'
# 读取配置文件
conf = utils.read_conf(argv_list[1])
# 遍历配置中要管理的目录
# 将需要进行备份的日志文件进行备份操作
# 并且将它们的路径记录到列表中，以备进行邮件提醒
backuped_logs_list = []
for path in conf['logs_path']:
# 获取当前path下日志文件路径及大小关系映射字典
logs_path_size_dict = utils.get_logs_path_size(path,
conf['logs_suffix'])
# 获取要进行备份的日志路径们
# （先进行配置文件中阈值数值单位转换）
backup_size_byte = utils.covert_unit_of_filesize(
conf['backup_size'], conf['unit'], 'B')
to_backup_logs_list = utils.get_to_backup_logs(logs_path_size_dict,
backup_size_byte)
# 遍历需要备份的日志文件，进行备份
for to_backup_log in to_backup_logs_list:
utils.backup_conf(to_backup_log)
# 记录到backuped_logs_list里
backuped_logs_list.append(to_backup_log)
# 判断是否需要email提醒
# 本次没有进行备份操作的话也不发送email
if conf['need_email'] and backuped_logs_list:
# 创建邮件内容
message = '''
你的{}服务器内，以下日志文件大小超过设置阈值，已进行备份：
{}
'''.format(conf['server_name'], backuped_logs_list)
# 发送电子邮件
utils.send_email(
from_email=conf['from_email'],
email_password=conf['email_password'],
to_email=conf['to_email'],
message=message,
)
def migrate(input_cfg, output_cfg):
d = utils.read_conf(input_cfg)
keys = list(default.keys())
keys.extend([
'hostname', 'ui_url_protocol', 'max_job_workers', 'customize_crt',
'ssl_cert', 'ssl_cert_key', 'secretkey_path', 'admiral_url',
'db_password', 'clair_db_password'
])
val = {}
for k in keys:
if k in d:
val[k] = d[k]
else:
val[k] = default[k]
tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.cfg.tpl')
utils.render(tpl_path, output_cfg, **val)
def migrate(input_cfg, output_cfg):
d = utils.read_conf(input_cfg)
val = {}
for k in keys:
val[k] = d.get(k, '')
#append registry to no_proxy
np_list = d.get('no_proxy', '').split(',')
if not 'registry' in np_list:
np_list.append('registry')
val['no_proxy'] = ','.join(np_list)
#handle harbor db information, if it previously pointed to internal mariadb, point it to the new default db instance of pgsql,
#update user to default pgsql user.
if 'mysql' == d['db_host']:
val['db_host'] = 'postgresql'
val['db_port'] = '5432'
val['db_user'] = '******'
#handle clair db information, if it pointed to internal pgsql in previous deployment, point it to the new default db instance of pgsql,
#the user should be the same user as harbor db
if 'postgres' == d['clair_db_host']:
val['clair_db_host'] = 'postgresql'
val['cliar_db_user'] = val['db_user']
val['clair_db_password'] = val['db_password']
tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.cfg.tpl')
utils.render(tpl_path, output_cfg, **val)
def migrate(input_cfg, output_cfg):
d = utils.read_conf(input_cfg)
val = {}
for k in keys:
val[k] = d.get(k,'')
#append registry to no_proxy
np_list = d.get('no_proxy','').split(',')
if not 'registry' in np_list:
np_list.append('registry')
val['no_proxy'] = ','.join(np_list)
#handle harbor db information, if it previously pointed to internal mariadb, point it to the new default db instance of pgsql,
#update user to default pgsql user.
if 'mysql' == d['db_host']:
val['db_host'] = 'postgresql'
val['db_port'] = '5432'
val['db_user'] = '******'
#handle clair db information, if it pointed to internal pgsql in previous deployment, point it to the new default db instance of pgsql,
#the user should be the same user as harbor db
if 'postgres' == d['clair_db_host']:
val['clair_db_host'] = 'postgresql'
val['cliar_db_user'] = val['db_user']
val['clair_db_password'] = val['db_password']
tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.cfg.tpl')
utils.render(tpl_path, output_cfg, **val)
def main(args):
args = read_conf(args.cfg, args)
torch.manual_seed(args.seed)
np.random.seed(args.seed)
if args.dataset == 'timit':
train_dataset = TIMIT(data_root=args.data_root,
datalist_root=args.datalist_root,
train=True,
oversampling=args.oversampling)
test_dataset = TIMIT(data_root=args.data_root,
datalist_root=args.datalist_root,
train=False)
elif args.dataset == 'libri':
raise NotImplementedError
else:
raise NotImplementedError
cost = nn.NLLLoss()
CNN_arch = {
'input_dim': train_dataset.wlen,
'fs': args.fs,
'cnn_N_filt': args.cnn_N_filt,
'cnn_len_filt': args.cnn_len_filt,
'cnn_max_pool_len': args.cnn_max_pool_len,
'cnn_use_laynorm_inp': args.cnn_use_laynorm_inp,
'cnn_use_batchnorm_inp': args.cnn_use_batchnorm_inp,
'cnn_use_laynorm': args.cnn_use_laynorm,
'cnn_use_batchnorm': args.cnn_use_batchnorm,
'cnn_act': args.cnn_act,
'cnn_drop': args.cnn_drop,
}
DNN1_arch = {
'fc_lay': args.fc_lay,
'fc_drop': args.fc_drop,
'fc_use_batchnorm': args.fc_use_batchnorm,
'fc_use_laynorm': args.fc_use_laynorm,
'fc_use_laynorm_inp': args.fc_use_laynorm_inp,
'fc_use_batchnorm_inp': args.fc_use_batchnorm_inp,
'fc_act': args.fc_act,
}
DNN2_arch = {
'input_dim': args.fc_lay[-1],
'fc_lay': args.class_lay,
'fc_drop': args.class_drop,
'fc_use_batchnorm': args.class_use_batchnorm,
'fc_use_laynorm': args.class_use_laynorm,
'fc_use_laynorm_inp': args.class_use_laynorm_inp,
'fc_use_batchnorm_inp': args.class_use_batchnorm_inp,
'fc_act': args.class_act,
}
model = SpeakerIDNet(CNN_arch, DNN1_arch, DNN2_arch)
if args.pt_file != '':
print("load model from:", args.pt_file)
checkpoint_load = torch.load(args.pt_file)
ext = os.path.splitext(args.pt_file)[1]
if ext == '.pkl':
model.load_raw_state_dict(checkpoint_load)
elif ext == '.pickle':
model.load_state_dict(checkpoint_load)
elif ext == '.pth':
load_checkpoint(model, args.pt_file)
else:
raise NotImplementedError
model = model.cuda()
if args.eval:
print('only eval the model')
evaluate(model, test_dataset, cost)
return
else:
print("train the model")
optimizer = optim.RMSprop(model.parameters(),
lr=args.lr,
alpha=0.95,
eps=1e-8)
train_dataloader = DataLoader(train_dataset,
args.batch_size,
shuffle=True,
num_workers=8,
pin_memory=True)
test_dataloader = DataLoader(test_dataset,
1,
shuffle=False,
num_workers=8,
pin_memory=True)
trainer = ClassifierTrainer(model,
train_dataloader,
optimizer,
cost,
batch_process,
args.output_dir,
0,
test_dataloader,
eval_every=args.N_eval_epoch,
print_every=args.print_every)
trainer.run(args.N_epochs)
### test (20210505_235209)
in_file = "data/wikitestko_from_table_not_h_4.jsonl"
out_file = f"output/test_out_ko_from_table_not_h_4_beam-{args.beam_size}_top-{args.topk}.jsonl"
label_file = "WikiSQL/data/test.jsonl"
db_file = "WikiSQL/data/test.db"
model_out_file = f"output/test_model_out_ko_from_table_not_h_4_beam-{args.beam_size}_top-{args.topk}.pkl"
###================================================================================================###
# All Best
model_path = "output/20210505_235209"
epoch = 4
engine = DBEngine(db_file)
config = utils.read_conf(os.path.join(model_path, "model.conf"))
# config["DEBUG"] = 1
featurizer = HydraFeaturizer(config)
pred_data = SQLDataset(in_file, config, featurizer, False)
print("num of samples: {0}".format(len(pred_data.input_features)))
##======================EG + TOP_k=============================##
model = create_model(config, is_train=False)
model.load(model_path, epoch)
if "DEBUG" in config:
model_out_file = model_out_file + ".partial"
if os.path.exists(model_out_file):
model_outputs = pickle.load(open(model_out_file, "rb"))
print(eval_file + ": " + result_str)
if "DEBUG" in self.config:
for text in sq:
print(text[0] + ":" + text[1] + "\t" + text[2])
else:
with open(self.eval_history_file, "a+", encoding="utf8") as f:
f.write("[{0}, epoch {1}] ".format(eval_file, epochs) +
result_str + "\n")
bad_case_file = os.path.join(
self.bad_case_dir,
"{0}_epoch_{1}.log".format(eval_file, epochs))
with open(bad_case_file, "w", encoding="utf8") as f:
for text in sq:
f.write(text[0] + ":" + text[1] + "\t" + text[2] +
"\n")
if __name__ == "__main__":
os.environ["CUDA_VISIBLE_DEVICES"] = "3"
config = utils.read_conf(os.path.join("conf", "wikisql.conf"))
config["DEBUG"] = 1
config["num_train_steps"] = 1000
config["num_warmup_steps"] = 100
featurizer = HydraFeaturizer(config)
model = create_model(config, is_train=True, num_gpu=1)
evaluator = HydraEvaluator("output", config, featurizer, model,
"debug evaluator")
evaluator.eval(0)
"--model_path",
help="trained model folder path (used in eval, predict and export mode)")
parser.add_argument(
"--epoch",
help="epochs to restore (used in eval, predict and export mode)")
parser.add_argument("--gpu", type=str, default=None, help="gpu id")
parser.add_argument("--note", type=str)
args = parser.parse_args()
if args.job == "train":
if args.gpu is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
# os.environ['TF_ENABLE_AUTO_MIXED_PRECISION'] = '1'
conf_path = os.path.abspath(args.conf)
config = utils.read_conf(conf_path)
note = args.note if args.note else ""
script_path = os.path.dirname(os.path.abspath(sys.argv[0]))
output_path = args.output_path
model_name = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
model_path = os.path.join(output_path, model_name)
if "DEBUG" not in config:
if not os.path.exists(output_path):
os.mkdir(output_path)
if not os.path.exists(model_path):
os.mkdir(model_path)
shutil.copyfile(conf_path, os.path.join(model_path, "model.conf"))
def execute_one_test(dataset, shot, model_moment, epoch):
os.environ["CUDA_VISIBLE_DEVICES"] = "0, 1, 2, 3"
model_path = "output/" + model_moment
in_file = "data/wiki{}_content.jsonl".format(
dataset) if shot == "orig" else "data/wiki{}_{}_content.jsonl".format(
dataset, shot)
db_file = "WikiSQL/data/{}.db".format(dataset)
label_file = "WikiSQL/data/{}.jsonl".format(
dataset) if shot == "orig" else "WikiSQL/data_{}/{}.jsonl".format(
shot, dataset)
out_path = "predictions/{}_{}_{}_{}".format(model_moment, epoch, dataset,
shot)
if not os.path.exists(out_path):
os.mkdir(out_path)
out_file = os.path.join(out_path, "out.jsonl")
eg_out_file = os.path.join(out_path, "out_eg.jsonl")
model_out_file = os.path.join(out_path, "model_out.pkl")
test_result_file = os.path.join(out_path, "result.txt")
engine = DBEngine(db_file)
config = utils.read_conf(os.path.join(model_path, "model.conf"))
# config["DEBUG"] = 1
featurizer = HydraFeaturizer(config)
pred_data = SQLDataset(in_file, config, featurizer, False)
print("num of samples: {0}".format(len(pred_data.input_features)))
model = create_model(config, is_train=False)
model.load(model_path, epoch)
if "DEBUG" in config:
model_out_file = model_out_file + ".partial"
model_outputs = model.dataset_inference(pred_data)
pickle.dump(model_outputs, open(model_out_file, "wb"))
# model_outputs = pickle.load(open(model_out_file, "rb"))
print("===HydraNet===")
pred_sqls = model.predict_SQL(pred_data, model_outputs=model_outputs)
with open(out_file, "w") as g:
for pred_sql in pred_sqls:
# print(pred_sql)
result = {"query": {}}
result["query"]["agg"] = int(pred_sql[0])
result["query"]["sel"] = int(pred_sql[1])
result["query"]["conds"] = [(int(cond[0]), int(cond[1]),
str(cond[2])) for cond in pred_sql[2]]
g.write(json.dumps(result) + "\n")
normal_res = print_metric(label_file, out_file, db_file)
print("===HydraNet+EG===")
pred_sqls = model.predict_SQL_with_EG(engine,
pred_data,
model_outputs=model_outputs)
with open(eg_out_file, "w") as g:
for pred_sql in pred_sqls:
# print(pred_sql)
result = {"query": {}}
result["query"]["agg"] = int(pred_sql[0])
result["query"]["sel"] = int(pred_sql[1])
result["query"]["conds"] = [(int(cond[0]), int(cond[1]),
str(cond[2])) for cond in pred_sql[2]]
g.write(json.dumps(result) + "\n")
eg_res = print_metric(label_file, eg_out_file, db_file)
with open(test_result_file, "w") as g:
g.write("normal results:\n" + normal_res + "eg results:\n" + eg_res)
def main(args):
speaker_cfg = args.speaker_cfg
speech_cfg = args.speech_cfg
args_speaker = read_conf(speaker_cfg, deepcopy(args))
args_speaker.model_path = args.speaker_model
args_speech = read_conf(speech_cfg, deepcopy(args))
args_speech.model_path = args.speech_model
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
print("set seed: ", args_speaker.optimization.seed)
torch.manual_seed(args_speaker.optimization.seed)
np.random.seed(args_speaker.optimization.seed)
random.seed(args_speaker.optimization.seed)
torch.cuda.set_device(args.local_rank)
if not args.no_dist:
torch.distributed.init_process_group(backend="nccl")
train_dataset = TIMIT_speaker(args.data_root,
train=True,
phoneme=True,
norm_factor=True)
test_dataset = TIMIT_speaker(args.data_root,
train=False,
phoneme=True,
norm_factor=True)
pretrained_models = get_pretrained_models(args_speaker, args_speech)
loss_factors = {
"speaker": args.speaker_factor,
"speech": args.speech_factor,
"norm": args.norm_factor
}
if args.target < 0: # non-targeted
speaker_loss = SpeakerLoss(pretrained_models['speaker'])
else: # targeted attack
speaker_loss = SpeakerLossTarget(pretrained_models['speaker'],
args.target)
loss_all = {}
loss_all['speech'] = {
'model':
pretrained_models['speech'],
'factor':
loss_factors['speech'],
'loss_func':
SpeechLoss(pretrained_models['speech'],
factor_kld=args.speech_kld_factor)
}
loss_all['speaker'] = {
'model': pretrained_models['speaker'],
'factor': loss_factors['speaker'],
'loss_func': speaker_loss
}
loss_all['norm'] = {
'loss_func': MSEWithThreshold(args.norm_clip),
'factor': loss_factors['norm']
}
cost = AdversarialLoss(loss_all)
model = SpeechTransformer(args.channel, args.kernel_size, args.dilation,
args.sample, args.noise_scale)
if args.pt_file != 'none':
print("load model from:", args.pt_file)
if os.path.splitext(args.pt_file)[1] == '.pkl':
checkpoint_load = torch.load(args.pt_file)
model.load_raw_state_dict(checkpoint_load)
else:
load_checkpoint(model, args.pt_file)
model = model.cuda()
if args.eval:
assert args.pt_file != 'none', "no pretrained model is provided!"
print('only eval the model')
evaluate(model, test_dataset, cost)
return
if args.test:
assert args.pt_file != 'none', "no pretrained model is provided!"
print("only test the model")
filename_list = open("./data/TIMIT/speaker/test.scp", 'r').readlines()
filename_list = [_f.strip() for _f in filename_list]
label_dict = np.load(
os.path.join(args.data_root, "processed",
"TIMIT_labels.npy")).item()
test_wav(model, filename_list, args.data_root,
os.path.join(args.data_root, "output"),
pretrained_models['speaker'], label_dict, args.target)
return
if args.cpu_test:
assert args.pt_file != 'none', "no pretrained model is provided!"
print("only cpu test the model")
filename_list = open("./data/TIMIT/speaker/test.scp", 'r').readlines()
filename_list = [_f.strip() for _f in filename_list]
label_dict = np.load(
os.path.join(args.data_root, "processed",
"TIMIT_labels.npy")).item()
test_wav_cpu(model, filename_list, args.data_root,
os.path.join(args.data_root, "output"),
pretrained_models['speaker'], label_dict, args.target)
return
print("train the model")
batch_process = batch_process_speaker
eval_hook = EvalHook()
optimizer = optim.Adam(model.parameters(),
lr=args_speaker.optimization.lr,
betas=(0.95, 0.999))
lr_scheduler = optim.lr_scheduler.StepLR(optimizer, 2, 0.5)
if args.no_dist:
kwarg = {
'shuffle':
True,
'worker_init_fn':
partial(_init_fn, seed=args_speaker.optimization.seed)
}
else:
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset)
kwarg = {
'sampler':
train_sampler,
'worker_init_fn':
partial(_init_fn, seed=args_speaker.optimization.seed)
}
train_dataloader = DataLoader(train_dataset,
args_speaker.optimization.batch_size,
num_workers=args.num_workers,
pin_memory=True,
**kwarg)
test_dataloader = DataLoader(test_dataset,
args_speaker.optimization.batch_size,
shuffle=False,
num_workers=args.num_workers,
pin_memory=True)
trainer = ClassifierTrainer(
model,
train_dataloader,
optimizer,
cost,
batch_process,
args.output_dir,
0,
test_dataloader,
eval_hook=eval_hook,
eval_every=args_speaker.optimization.N_eval_epoch,
print_every=args_speaker.optimization.print_every,
lr_scheduler=lr_scheduler)
trainer.logger.info(args)
trainer.run(args_speaker.optimization.N_epochs)
import tweepy
def inputs():
keyword = raw_input("Enter keyword to test: ")
return keyword
#def checkFilter(keywords, lines):
# while keywords in lines:
if __name__=="__main__":
try:
scrape()
cfs = ChiFeatureSelector('trending.%d.json'%os.getpid(), 'nontrending.%d.ujson'%os.getpid())
except:
classify = classifier.HashtagClassifier()
classify.condProb = utils.read_conf('classifierTrained.json')
classify.prior = utils.read_conf('classifier_prior.json')
while True:
keyword = re.sub("""[\s/:*"<>?|\\.;'\[\]]+""", '', inputs())
if not keyword:
print 'Please enter a valid phrase'
continue
try:
scrapeTrends.search_tweet(keyword)
except tweepy.TweepError:
print 'Please enter a valid phrase'
continue
try:
tweets = utils.read_tweets('tweets/tweets.%(name)s.json'%{'name':keyword})
except:
