def process_toc(self):
for addon in os.listdir('AddOns'):
config = self.get_addon_config(addon)
if not config:
logger.warning('%s not found!', addon)
continue
path = os.path.join('AddOns', addon, '{}.toc'.format(addon))
def process(config, addon, lines):
toc = TOC(lines)
if self.is_classic:
toc.tags['Interface'] = CLASSIC_VER
else:
toc.tags['Interface'] = RETAIL_VER
toc.tags['Title-zhCN'] = self.get_title(addon)
ns = {'x': 'https://www.github.com/luhao007'}
note = config.find('x:Notes', ns)
if note is not None:
toc.tags['Notes-zhCN'] = note.text
if config.tag.endswith('SubAddon'):
parent_config = self.get_addon_parent_config(addon)
toc.tags['X-Part-Of'] = parent_config.get('name')
elif addon in ['DBM-Core', 'Auc-Advanced', 'TomCats']:
toc.tags['X-Part-Of'] = addon
elif addon in ['+Wowhead_Looter']:
toc.tags.pop('X-Part-Of', None)
return toc.to_lines()
process_file(path, functools.partial(process, config, addon))
def handle_lib_graph(self):
def handle_graph(lines):
orig = 'local TextureDirectory\n'
tar = 'local TextureDirectory = "Interface\\\\AddOns\\\\!!Libs' + \
'\\\\LibGraph-2.0\\\\LibGraph-2.0\\\\"\n'
if orig not in lines:
return lines
# Comment out the block discovering TextureDirectory
# The dictory discovery system is not working here
start = lines.index(orig)
ret = lines[:start]
ret.append(tar)
end = lines[start:].index('end\n')
for line in lines[start + 1:start + end + 1]:
ret.append('--{}'.format(line))
ret += lines[start + end + 1:]
return ret
process_file(
'AddOns/!!Libs/LibGraph-2.0/LibGraph-2.0/LibGraph-2.0.lua',
handle_graph)
def handle_monkeyspeed(self):
process_file(
'AddOns/MonkeySpeed/MonkeySpeedInit.lua', lambda lines: [
line.replace('GetAddOnMetadata("MonkeySpeed", "Title")',
'"MonkeySpeed"') if '"Title"' in line else line
for line in lines
])
def remove_libraries(self, libs, root, xml_path):
"""Remove selected embedded libraries from root and xml."""
for lib in libs:
rm_tree(Path(root) / lib)
process_file(
xml_path, lambda lines: [
line for line in lines
if not any(lib + '\\' in line for lib in libs)
])
def handle_bagnon(self):
rm_tree('AddOns/Bagnon/common/LibDataBroker-1.1')
process_file(
'AddOns/Bagnon/AddOns/main/main.xml', lambda lines:
[line for line in lines if 'LibDataBroker' not in line])
self.remove_libraries([
'AceEvent-3.0', 'AceLocale-3.0', 'CallbackHandler-1.0', 'LibStub'
], 'AddOns/Bagnon/common/Wildpants/libs',
'AddOns/Bagnon/common/Wildpants/libs/libs.xml')
def handle_oa(self):
self.remove_libraries([
'CallbackHandler-1.0', 'LibBabble-Inventory-3.0',
'LibBabble-SubZone-3.0', 'LibSharedMedia-3.0', 'LibStub'
], 'Addons/Overachiever/libs', 'Addons/Overachiever/Overachiever.toc')
process_file(
'Addons/Overachiever/Overachiever.lua', lambda lines: [
line.replace('GetAddOnMetadata("Overachiever", "Title")',
'"Overarchiever"') for line in lines
])
def handle_decursive(self):
for lib in os.listdir('AddOns/Decursive/Libs'):
if lib == 'BugGrabber':
continue
rm_tree(Path('AddOns/Decursive/Libs') / lib)
process_file(
'AddOns/Decursive/embeds.xml', lambda lines: [
line for line in lines
if 'Libs' not in line or 'BugGrabber' in line
])
def change_defaults(self, path, defaults):
def handle(lines):
ret = []
for line in lines:
for d in [defaults] if isinstance(defaults, str) else defaults:
if line.startswith(d.split('= ')[0] + '= '):
ret.append(d + '\n')
break
else:
ret.append(line)
return ret
process_file(path, handle)
def handle_auctioneer(self):
addons = ['Auc-Advanced', 'BeanCounter', 'Enchantrix', 'Informant']
for addon in addons:
rm_tree(Path('AddOns') / addon / 'Libs' / 'LibDataBroker')
process_file(
Path('AddOns') / addon / 'Libs' / 'Load.xml', lambda lines:
[line for line in lines if 'LibDataBroker' not in line])
rm_tree('AddOns/SlideBar/Libs')
process_file(
'AddOns/SlideBar/Load.xml',
lambda lines: [line for line in lines if 'Libs' not in line])
def remove_libs_in_file(self, path, libs):
def f(lines):
if str(path).endswith('.toc'):
pattern = r'(?i)\s*{}.*'
else:
pattern = r'\s*<((Script)|(Include))+ file\s*=\s*"{}.*'
return [
line for line in lines if not any(
re.match(pattern.format(lib), line) for lib in libs)
]
process_file(path, f)
def handle_tc2(self):
path = 'Addons/ThreatClassic2/Libs/LibThreatClassic2'
if os.path.exists(path):
dst = 'Addons/!!Libs/LibThreatClassic2'
rm_tree(dst)
shutil.copytree(path, dst)
rm_tree('AddOns/ThreatClassic2/Libs')
def f(lines):
return [line for line in lines if 'Libs' not in line]
path = 'AddOns/ThreatClassic2/ThreatClassic2.xml'
process_file(path, f)
def handle_fizzle(self):
def f(lines):
ret = []
for line in lines:
if line == ' DisplayWhenFull = true,':
# Change default settings to not dixplay the '100%'.
ret.append(' DisplayWhenFull = false,')
else:
# Only show 'Fizzle' in the options, not the meta name.
ret.append(
line.replace('GetAddOnMetadata("Fizzle", "Title")',
'"Fizzle"'))
return ret
process_file('AddOns/Fizzle/Core.lua', f)
def handle_btwquest(self):
def f(lines):
ret = []
minimap = False
for line in lines:
if 'minimapShown' in line:
minimap = True
if minimap and 'default' in line:
ret.append(' default = false,\n')
minimap = False
else:
ret.append(line)
return ret
process_file('Addons/BtWQuests/BtWQuests.lua', f)
def handle_acp(self):
def handle(lines):
ret = []
start1 = start2 = 0
for i, l in enumerate(lines):
if 'FontString name="$parentTitle"' in l:
start1 = i
elif 'FontString name="$parentStatus"' in l:
start2 = i
ret = lines[:start1 + 2]
ret.append(' ' * 24 + '<AbsDimension x="270" y="12"/>\n')
ret += lines[start1 + 3:start2 + 2]
ret.append(' ' * 24 + '<AbsDimension x="90" y="12"/>\n')
ret += lines[start2 + 3:]
return ret
process_file('Addons/ACP/ACP.xml', handle)
def handle_ufp(self):
if self.is_classic:
rm_tree('AddOns/UnitFramesPlus_MobHealth')
self.remove_libraries_all('UnitFramesPlus_Cooldown')
def f(lines):
ret = []
minimap = False
for line in lines:
if 'minimap = {' in line:
minimap = True
if minimap and 'button' in line:
ret.append(' button = 0,\n')
minimap = False
else:
ret.append(line)
return ret
process_file('Addons/UnitFramesPlus/UnitFramesPlus.lua', f)
def handle_questie(self):
self.remove_libraries([
'AceAddon-3.0', 'AceBucket-3.0', 'AceComm-3.0', 'AceConfig-3.0',
'AceConsole-3.0', 'AceDB-3.0', 'AceDBOptions-3.0', 'AceEvent-3.0',
'AceGUI-3.0', 'AceGUI-3.0-SharedMediaWidgets', 'AceHook-3.0',
'AceLocale-3.0', 'AceSerializer-3.0', 'AceTab-3.0', 'AceTimer-3.0',
'CallbackHandler-1.0', 'LibCompress', 'LibDataBroker-1.1',
'LibDBIcon-1.0', 'LibSharedMedia', 'LibStub'
], 'AddOns/Questie/Libs', 'AddOns/Questie/embeds.xml')
root = Path('AddOns/Questie')
with open(root / 'Questie.toc', 'r', encoding='utf-8') as f:
lines = f.readlines()
toc = TOC(lines)
version = toc.tags['Version']
major, minor, patch = version.split(' ')[0].split('.')
def handle(lines):
func = 'function QuestieLib:GetAddonVersionInfo()'
start = 0
for i, line in enumerate(lines):
if line.startswith(func):
start = i
break
ret = lines[:start + 1]
ret.append(' return {}, {}, {}\n'.format(major, minor, patch))
ret.append('end\n')
end = lines[start:].index('end\n')
if not lines[start + 1].strip().startswith('return'):
for line in lines[start + 1:start + end + 1]:
ret.append('--{}'.format(line))
ret += lines[start + end + 1:]
return ret
process_file(root / 'Modules/Libs/QuestieLib.lua', handle)
def main(shield_params, n_events, first_event, file_name, step_geo,
const_field):
"""
Gets vector of optimised shield parameters(not full one), run SHiP simulation
saves dict of magnet length, weight, optimised_paramteters, input kinematics and output hit distribution
:return:
"""
shield_params = np.array(
[float(x.strip()) for x in shield_params.split(',')], dtype=float)
ship_runner = SHIPRunner(GEOFILE,
file_name=file_name,
step_geo=step_geo,
const_field=const_field)
fair_runner = ship_runner.run_ship(n_events=n_events,
first_event=first_event)
gm = GeometryManipulator()
l, w, tracker_ends = gm.extract_l_and_w(GEOFILE, "full_ship_geofile.root",
fair_runner)
muons_stats = process_file(ship_runner.output_file,
tracker_ends,
apply_acceptance_cut=True,
debug=False)
if len(muons_stats) == 0:
veto_points, muon_kinematics = np.array([]), np.array([])
else:
veto_points = muons_stats[:, -2:]
muon_kinematics = muons_stats[:, :-2]
returned_params = {
"l": l,
"w": w,
"params": shield_params.tolist(),
"kinematics": muon_kinematics.tolist(),
"veto_points": veto_points.tolist()
}
with open(os.path.join(ship_runner.output_dir, "optimise_input.json"),
"w") as f:
json.dump(returned_params, f)
def process_to_csv(path, out_dir, event_sources, add_types=None):
"""
Process file and saves results to CSV in output directory.
:param path: Path to input text file
:param out_dir: Directory to save cleaned file
:param event_sources: Names of events to include
:param add_types: Additional columns to include beyond those defined by the event sources.
"""
print("Cleaning ", path)
base_name = os.path.basename(path)
try:
df = process_file(path, event_sources, add_types=add_types)
out_path = out_dir + os.path.splitext(base_name)[0] + '.csv'
# TODO: Move this test up into main filter function
df[df['SlideType'] == 'TestImage'].to_csv(out_path, index=False)
except Exception as e:
print("Could not process ", path, file=sys.stderr)
print(e, file=sys.stderr)
print("Events found: ", read_events(path), file=sys.stderr)
print("Columns found: ", read_cols(path), file=sys.stderr)
# -*- coding: utf-8 -*-
'''
@authors: Daniel Reyes, Ania Pietrzak, Alberto Miedes
'''
'''
Implementar un programa en Python tal que tomando como entrada el nombre de un
archivo de texto, analice el contenido y escriba otro archivo de salida que
contenga el número de veces que aparece cada palabra.
'''
import utils
# Pedir el nombre del fichero
filename = raw_input("Nombre del fichero: ")
# process file
utils.process_file(filename)
def on_extract_clicked(self):
# print(self.tree_dict)
# self.find_code("")
#
if len(self.tree_dict) == 0:
QMessageBox.information(self, "提示", ' 未生成查找树！ ')
return
if self.pathEdit.text().strip() == "" or self.targetEdit.text().strip(
) == "":
QMessageBox.information(self, "提示", ' 文件路径和目标路径不能为空！ ')
return
if self.pageEdit.text().strip() == "":
QMessageBox.information(self, "提示", ' 页码不能为空！ ')
return
if self.budgetEdit.text().strip() == "" and self.actualEdit.text(
).strip() == "":
QMessageBox.information(self, "提示", ' 预算数、决算数不能同时为空！ ')
return
if self.subEdit.text().strip() == "" and self.codeEdit.text().strip(
) == "":
QMessageBox.information(self, "提示", ' 科目名称和科目编码不能同时为空！ ')
return
if self.subEdit.text().strip() != "" and self.codeEdit.text().strip(
) != "":
QMessageBox.information(self, "提示", ' 科目名称和科目编码不能同时非空！ ')
return
try:
process_file(self.pathEdit.text().strip())
suffix = self.pathEdit.text().split(".")[-1]
# print(self.pathEdit.text())
page = process_param(self.pageEdit.text().strip())
if suffix.lower() == "doc" or suffix.lower(
) == "docx" or suffix.lower() == "pdf":
bound_info = self.boundEdit.text().strip()
print(bound_info)
if bound_info != "":
if bound_info.endswith(","):
bound_info = bound_info.rstrip(",")
# print(page)
if len(page) > 0:
# print((list(map(str, page))))
pdf = camelot.read_pdf(self.pathEdit.text().strip(),
flavor='stream',
pages=','.join(
list(map(str, page))),
table_areas=[bound_info])
print(len(pdf))
for i in range(len(pdf)):
table_list = []
for row_data in pdf[i].df.values.tolist():
table_list.append(row_data)
self.parse(table_list)
QMessageBox.information(self, "提示", ' 提取信息结束！ ')
print(self.json)
else:
"""
pdf = pdfplumber.open(self.pathEdit.text().strip())
# print(pdf.pages)
# print(len(pdf.pages))
# print(page)
for i, _page in enumerate(pdf.pages):
if i + 1 in page:
table_list = []
print(_page.extract_text())
for pdf_table in _page.extract_tables():
for row in pdf_table:
table_list.append(row)
print(row)
print(table_list)
self.parse(table_list)
"""
if len(page) > 0:
# print((list(map(str, page))))
if suffix.lower() == "doc" or suffix.lower() == "docx":
pdf = camelot.read_pdf(
self.pathEdit.text().strip().replace(
"." + suffix, ".pdf"),
flavor='stream',
pages=','.join(list(map(str, page))))
else:
pdf = camelot.read_pdf(
self.pathEdit.text().strip(),
flavor='stream',
pages=','.join(list(map(str, page))))
# print(len(pdf))
for i in range(len(pdf)):
table_list = []
for row_data in pdf[i].df.values.tolist():
table_list.append(row_data)
self.parse(table_list)
QMessageBox.information(self, "提示", ' 提取信息结束！ ')
elif suffix.lower() == "xls" or suffix.lower() == "xlsx":
wb = xlrd.open_workbook(self.pathEdit.text().strip())
sheet_names = wb.sheet_names()
for i, sheet_name in enumerate(sheet_names):
# print(sheet_name)
if i + 1 in page:
table = wb.sheet_by_index(i)
table_list = []
print(table.nrows)
for ii in range(table.nrows):
# print(type(table.row_values(ii)))
# print(table.row_values(ii))
table_list.append(table.row_values(ii))
print(table_list)
self.parse(table_list)
QMessageBox.information(self, "提示", ' 提取信息结束！ ')
except:
QMessageBox.information(self, "提示", "提取信息失败请查看输入是否有误！")
from utils import process_file, ResultMeasurements
def print_results(result_measurements: ResultMeasurements):
print(
f'Hottest day is {result_measurements.hottest_day.date} with temperature '
f'{result_measurements.hottest_day.temperature}')
print(
f'Coldest day is {result_measurements.coldest_day.date} with temperature '
f'{result_measurements.coldest_day.temperature}')
print(
f'Hottest month is {result_measurements.hottest_month.date.strftime("%Y-%m")} with temperature '
f'{result_measurements.hottest_month.temperature}')
print(
f'Coldest month is {result_measurements.coldest_month.date.strftime("%Y-%m")} with temperature '
f'{result_measurements.coldest_month.temperature}')
if __name__ == '__main__':
filename = '34300.01.01.2016.01.01.2017.1.0.0.ru.csv'
print_results(process_file(filename))
def evaluate(actor_critic, ob_rms, env_name, act_func, seed, k):
num_stack = 1
log_interval = 1
load_dir = './trained_models/ppo/'
#sd = np.random.randint(1000, size=1)
sd = [1]
env = make_env(env_name, int(sd[0]), 0,
'./tmp/test/' + str(seed) + '_' + act_func + '_')
env = DummyVecEnv([env])
#actor_critic, ob_rms = \
# torch.load(os.path.join(load_dir, 'ppo_'+env_name+'_'+act_func+ '_'+str(seed)+".pt"))
if len(env.observation_space.shape) == 1:
env = VecNormalize(env, ret=False)
env.ob_rms = ob_rms
# An ugly hack to remove updates
def _obfilt(self, obs):
if self.ob_rms:
obs = np.clip((obs - self.ob_rms.mean) /
np.sqrt(self.ob_rms.var + self.epsilon),
-self.clipob, self.clipob)
return obs
else:
return obs
env._obfilt = types.MethodType(_obfilt, env)
render_func = env.venv.envs[0].render
else:
render_func = env.envs[0].render
obs_shape = env.observation_space.shape
obs_shape = (obs_shape[0] * num_stack, *obs_shape[1:])
current_obs = torch.zeros(1, *obs_shape)
states = torch.zeros(1, actor_critic.state_size)
masks = torch.zeros(1, 1)
if torch.cuda.is_available():
current_obs = current_obs.cuda()
states = states.cuda()
masks = masks.cuda()
def update_current_obs(obs):
shape_dim0 = env.observation_space.shape[0]
obs = torch.from_numpy(obs).float()
if num_stack > 1:
current_obs[:, :-shape_dim0] = current_obs[:, shape_dim0:]
current_obs[:, -shape_dim0:] = obs
#render_func('human')
obs = env.reset()
update_current_obs(obs)
if env_name.find('Bullet') > -1:
import pybullet as p
torsoId = -1
for i in range(p.getNumBodies()):
if (p.getBodyInfo(i)[0].decode() == "torso"):
torsoId = i
for i in range(5000):
value, action, _, states = actor_critic.act(Variable(current_obs,
volatile=True),
Variable(states,
volatile=True),
Variable(masks,
volatile=True),
deterministic=True)
#if i % 1000 == 0:
# print("STEP: ", i)
states = states.data
cpu_actions = action.data.squeeze(1).cpu().numpy()
# Obser reward and next obs
obs, reward, done, _ = env.step(cpu_actions)
masks.fill_(0.0 if done else 1.0)
if current_obs.dim() == 4:
current_obs *= masks.unsqueeze(2).unsqueeze(2)
else:
current_obs *= masks
update_current_obs(obs)
if env_name.find('Bullet') > -1:
if torsoId > -1:
distance = 5
yaw = 0
humanPos, humanOrn = p.getBasePositionAndOrientation(torsoId)
p.resetDebugVisualizerCamera(distance, yaw, -20, humanPos)
#render_func('human')
#print(dir(env.envs[0]))
process_file(env_name, act_func, sd[0], seed, k)
def train():
print("Configuring TensorBoard and Saver...")
# 配置 Tensorboard，重新训练时，请将tensorboard文件夹删除，不然图会覆盖
tensorboard_dir = 'tensorboard/textcnn'
if not os.path.exists(tensorboard_dir):
os.makedirs(tensorboard_dir)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
print("Loading training and validation data...")
# 载入训练集与验证集
start_time = time.time()
x_train, y_train = process_file(train_dir, word_to_id, cat_to_id,
config.seq_length)
x_val, y_val = process_file(val_dir, word_to_id, cat_to_id,
config.seq_length)
time_dif = get_time_dif(start_time)
print("Time usage:", time_dif)
# 创建session
with model.graph.as_default():
tf.summary.scalar("loss", model.loss)
tf.summary.scalar("accuracy", model.acc)
merged_summary = tf.summary.merge_all()
writer = tf.summary.FileWriter(tensorboard_dir)
saver = tf.train.Saver()
model.session.run(tf.global_variables_initializer())
writer.add_graph(model.graph)
print('Training and evaluating...')
start_time = time.time()
total_batch = 0 # 总批次
best_acc_val = 0.0 # 最佳验证集准确率
last_improved = 0 # 记录上一次提升批次
require_improvement = 1000 # 如果超过1000轮未提升，提前结束训练
flag = False
for epoch in range(config.num_epochs):
print('Epoch:', epoch + 1)
batch_train = batch_iter(x_train, y_train, config.batch_size)
for x_batch, y_batch in batch_train:
feed_dict = feed_data(x_batch, y_batch,
config.dropout_keep_prob)
if total_batch % config.save_per_batch == 0:
# 每多少轮次将训练结果写入tensorboard scalar
s = model.session.run(merged_summary, feed_dict=feed_dict)
writer.add_summary(s, total_batch)
if total_batch % config.print_per_batch == 0:
# 每多少轮次输出在训练集和验证集上的性能
feed_dict[model.keep_prob] = 1.0
loss_train, acc_train = model.session.run(
[model.loss, model.acc], feed_dict=feed_dict)
loss_val, acc_val = evaluate(model.session, x_val,
y_val) # todo
if acc_val > best_acc_val:
# 保存最好结果
best_acc_val = acc_val
last_improved = total_batch
saver.save(sess=model.session, save_path=save_path)
improved_str = '*'
else:
improved_str = ''
time_dif = get_time_dif(start_time)
msg = 'Iter: {0:>6}, Train Loss: {1:>6.2}, Train Acc: {2:>7.2%},' \
+ ' Val Loss: {3:>6.2}, Val Acc: {4:>7.2%}, Time: {5} {6}'
print(
msg.format(total_batch, loss_train, acc_train,
loss_val, acc_val, time_dif, improved_str))
model.session.run(model.optim, feed_dict=feed_dict) # 运行优化
total_batch += 1
if total_batch - last_improved > require_improvement:
# 验证集正确率长期不提升，提前结束训练
print("No optimization for a long time, auto-stopping...")
flag = True
break # 跳出循环
if flag: # 同上
break
def handle_tsm(self):
rm_tree('AddOns/TradeSkillMaster/External/EmbeddedLibs/')
process_file(
'AddOns/TradeSkillMaster/TradeSkillMaster.toc', lambda lines:
[line for line in lines if 'EmbeddedLibs' not in line])
def handle_omnicc(self):
process_file(
'AddOns/OmniCC/core/core.xml',
lambda lines: [line for line in lines if 'libs' not in line])
