def process_toc(self): for addon in os.listdir('AddOns'): config = self.get_addon_config(addon) if not config: logger.warning('%s not found!', addon) continue path = os.path.join('AddOns', addon, '{}.toc'.format(addon)) def process(config, addon, lines): toc = TOC(lines) if self.is_classic: toc.tags['Interface'] = CLASSIC_VER else: toc.tags['Interface'] = RETAIL_VER toc.tags['Title-zhCN'] = self.get_title(addon) ns = {'x': 'https://www.github.com/luhao007'} note = config.find('x:Notes', ns) if note is not None: toc.tags['Notes-zhCN'] = note.text if config.tag.endswith('SubAddon'): parent_config = self.get_addon_parent_config(addon) toc.tags['X-Part-Of'] = parent_config.get('name') elif addon in ['DBM-Core', 'Auc-Advanced', 'TomCats']: toc.tags['X-Part-Of'] = addon elif addon in ['+Wowhead_Looter']: toc.tags.pop('X-Part-Of', None) return toc.to_lines() process_file(path, functools.partial(process, config, addon))
def handle_lib_graph(self): def handle_graph(lines): orig = 'local TextureDirectory\n' tar = 'local TextureDirectory = "Interface\\\\AddOns\\\\!!Libs' + \ '\\\\LibGraph-2.0\\\\LibGraph-2.0\\\\"\n' if orig not in lines: return lines # Comment out the block discovering TextureDirectory # The dictory discovery system is not working here start = lines.index(orig) ret = lines[:start] ret.append(tar) end = lines[start:].index('end\n') for line in lines[start + 1:start + end + 1]: ret.append('--{}'.format(line)) ret += lines[start + end + 1:] return ret process_file( 'AddOns/!!Libs/LibGraph-2.0/LibGraph-2.0/LibGraph-2.0.lua', handle_graph)
def handle_monkeyspeed(self): process_file( 'AddOns/MonkeySpeed/MonkeySpeedInit.lua', lambda lines: [ line.replace('GetAddOnMetadata("MonkeySpeed", "Title")', '"MonkeySpeed"') if '"Title"' in line else line for line in lines ])
def remove_libraries(self, libs, root, xml_path): """Remove selected embedded libraries from root and xml.""" for lib in libs: rm_tree(Path(root) / lib) process_file( xml_path, lambda lines: [ line for line in lines if not any(lib + '\\' in line for lib in libs) ])
def handle_bagnon(self): rm_tree('AddOns/Bagnon/common/LibDataBroker-1.1') process_file( 'AddOns/Bagnon/AddOns/main/main.xml', lambda lines: [line for line in lines if 'LibDataBroker' not in line]) self.remove_libraries([ 'AceEvent-3.0', 'AceLocale-3.0', 'CallbackHandler-1.0', 'LibStub' ], 'AddOns/Bagnon/common/Wildpants/libs', 'AddOns/Bagnon/common/Wildpants/libs/libs.xml')
def handle_oa(self): self.remove_libraries([ 'CallbackHandler-1.0', 'LibBabble-Inventory-3.0', 'LibBabble-SubZone-3.0', 'LibSharedMedia-3.0', 'LibStub' ], 'Addons/Overachiever/libs', 'Addons/Overachiever/Overachiever.toc') process_file( 'Addons/Overachiever/Overachiever.lua', lambda lines: [ line.replace('GetAddOnMetadata("Overachiever", "Title")', '"Overarchiever"') for line in lines ])
def handle_decursive(self): for lib in os.listdir('AddOns/Decursive/Libs'): if lib == 'BugGrabber': continue rm_tree(Path('AddOns/Decursive/Libs') / lib) process_file( 'AddOns/Decursive/embeds.xml', lambda lines: [ line for line in lines if 'Libs' not in line or 'BugGrabber' in line ])
def change_defaults(self, path, defaults): def handle(lines): ret = [] for line in lines: for d in [defaults] if isinstance(defaults, str) else defaults: if line.startswith(d.split('= ')[0] + '= '): ret.append(d + '\n') break else: ret.append(line) return ret process_file(path, handle)
def handle_auctioneer(self): addons = ['Auc-Advanced', 'BeanCounter', 'Enchantrix', 'Informant'] for addon in addons: rm_tree(Path('AddOns') / addon / 'Libs' / 'LibDataBroker') process_file( Path('AddOns') / addon / 'Libs' / 'Load.xml', lambda lines: [line for line in lines if 'LibDataBroker' not in line]) rm_tree('AddOns/SlideBar/Libs') process_file( 'AddOns/SlideBar/Load.xml', lambda lines: [line for line in lines if 'Libs' not in line])
def remove_libs_in_file(self, path, libs): def f(lines): if str(path).endswith('.toc'): pattern = r'(?i)\s*{}.*' else: pattern = r'\s*<((Script)|(Include))+ file\s*=\s*"{}.*' return [ line for line in lines if not any( re.match(pattern.format(lib), line) for lib in libs) ] process_file(path, f)
def handle_tc2(self): path = 'Addons/ThreatClassic2/Libs/LibThreatClassic2' if os.path.exists(path): dst = 'Addons/!!Libs/LibThreatClassic2' rm_tree(dst) shutil.copytree(path, dst) rm_tree('AddOns/ThreatClassic2/Libs') def f(lines): return [line for line in lines if 'Libs' not in line] path = 'AddOns/ThreatClassic2/ThreatClassic2.xml' process_file(path, f)
def handle_fizzle(self): def f(lines): ret = [] for line in lines: if line == ' DisplayWhenFull = true,': # Change default settings to not dixplay the '100%'. ret.append(' DisplayWhenFull = false,') else: # Only show 'Fizzle' in the options, not the meta name. ret.append( line.replace('GetAddOnMetadata("Fizzle", "Title")', '"Fizzle"')) return ret process_file('AddOns/Fizzle/Core.lua', f)
def handle_btwquest(self): def f(lines): ret = [] minimap = False for line in lines: if 'minimapShown' in line: minimap = True if minimap and 'default' in line: ret.append(' default = false,\n') minimap = False else: ret.append(line) return ret process_file('Addons/BtWQuests/BtWQuests.lua', f)
def handle_acp(self): def handle(lines): ret = [] start1 = start2 = 0 for i, l in enumerate(lines): if 'FontString name="$parentTitle"' in l: start1 = i elif 'FontString name="$parentStatus"' in l: start2 = i ret = lines[:start1 + 2] ret.append(' ' * 24 + '<AbsDimension x="270" y="12"/>\n') ret += lines[start1 + 3:start2 + 2] ret.append(' ' * 24 + '<AbsDimension x="90" y="12"/>\n') ret += lines[start2 + 3:] return ret process_file('Addons/ACP/ACP.xml', handle)
def handle_ufp(self): if self.is_classic: rm_tree('AddOns/UnitFramesPlus_MobHealth') self.remove_libraries_all('UnitFramesPlus_Cooldown') def f(lines): ret = [] minimap = False for line in lines: if 'minimap = {' in line: minimap = True if minimap and 'button' in line: ret.append(' button = 0,\n') minimap = False else: ret.append(line) return ret process_file('Addons/UnitFramesPlus/UnitFramesPlus.lua', f)
def handle_questie(self): self.remove_libraries([ 'AceAddon-3.0', 'AceBucket-3.0', 'AceComm-3.0', 'AceConfig-3.0', 'AceConsole-3.0', 'AceDB-3.0', 'AceDBOptions-3.0', 'AceEvent-3.0', 'AceGUI-3.0', 'AceGUI-3.0-SharedMediaWidgets', 'AceHook-3.0', 'AceLocale-3.0', 'AceSerializer-3.0', 'AceTab-3.0', 'AceTimer-3.0', 'CallbackHandler-1.0', 'LibCompress', 'LibDataBroker-1.1', 'LibDBIcon-1.0', 'LibSharedMedia', 'LibStub' ], 'AddOns/Questie/Libs', 'AddOns/Questie/embeds.xml') root = Path('AddOns/Questie') with open(root / 'Questie.toc', 'r', encoding='utf-8') as f: lines = f.readlines() toc = TOC(lines) version = toc.tags['Version'] major, minor, patch = version.split(' ')[0].split('.') def handle(lines): func = 'function QuestieLib:GetAddonVersionInfo()' start = 0 for i, line in enumerate(lines): if line.startswith(func): start = i break ret = lines[:start + 1] ret.append(' return {}, {}, {}\n'.format(major, minor, patch)) ret.append('end\n') end = lines[start:].index('end\n') if not lines[start + 1].strip().startswith('return'): for line in lines[start + 1:start + end + 1]: ret.append('--{}'.format(line)) ret += lines[start + end + 1:] return ret process_file(root / 'Modules/Libs/QuestieLib.lua', handle)
def main(shield_params, n_events, first_event, file_name, step_geo, const_field): """ Gets vector of optimised shield parameters(not full one), run SHiP simulation saves dict of magnet length, weight, optimised_paramteters, input kinematics and output hit distribution :return: """ shield_params = np.array( [float(x.strip()) for x in shield_params.split(',')], dtype=float) ship_runner = SHIPRunner(GEOFILE, file_name=file_name, step_geo=step_geo, const_field=const_field) fair_runner = ship_runner.run_ship(n_events=n_events, first_event=first_event) gm = GeometryManipulator() l, w, tracker_ends = gm.extract_l_and_w(GEOFILE, "full_ship_geofile.root", fair_runner) muons_stats = process_file(ship_runner.output_file, tracker_ends, apply_acceptance_cut=True, debug=False) if len(muons_stats) == 0: veto_points, muon_kinematics = np.array([]), np.array([]) else: veto_points = muons_stats[:, -2:] muon_kinematics = muons_stats[:, :-2] returned_params = { "l": l, "w": w, "params": shield_params.tolist(), "kinematics": muon_kinematics.tolist(), "veto_points": veto_points.tolist() } with open(os.path.join(ship_runner.output_dir, "optimise_input.json"), "w") as f: json.dump(returned_params, f)
def process_to_csv(path, out_dir, event_sources, add_types=None): """ Process file and saves results to CSV in output directory. :param path: Path to input text file :param out_dir: Directory to save cleaned file :param event_sources: Names of events to include :param add_types: Additional columns to include beyond those defined by the event sources. """ print("Cleaning ", path) base_name = os.path.basename(path) try: df = process_file(path, event_sources, add_types=add_types) out_path = out_dir + os.path.splitext(base_name)[0] + '.csv' # TODO: Move this test up into main filter function df[df['SlideType'] == 'TestImage'].to_csv(out_path, index=False) except Exception as e: print("Could not process ", path, file=sys.stderr) print(e, file=sys.stderr) print("Events found: ", read_events(path), file=sys.stderr) print("Columns found: ", read_cols(path), file=sys.stderr)
# -*- coding: utf-8 -*- ''' @authors: Daniel Reyes, Ania Pietrzak, Alberto Miedes ''' ''' Implementar un programa en Python tal que tomando como entrada el nombre de un archivo de texto, analice el contenido y escriba otro archivo de salida que contenga el número de veces que aparece cada palabra. ''' import utils # Pedir el nombre del fichero filename = raw_input("Nombre del fichero: ") # process file utils.process_file(filename)
def on_extract_clicked(self): # print(self.tree_dict) # self.find_code("") # if len(self.tree_dict) == 0: QMessageBox.information(self, "提示", ' 未生成查找树! ') return if self.pathEdit.text().strip() == "" or self.targetEdit.text().strip( ) == "": QMessageBox.information(self, "提示", ' 文件路径和目标路径不能为空! ') return if self.pageEdit.text().strip() == "": QMessageBox.information(self, "提示", ' 页码不能为空! ') return if self.budgetEdit.text().strip() == "" and self.actualEdit.text( ).strip() == "": QMessageBox.information(self, "提示", ' 预算数、决算数不能同时为空! ') return if self.subEdit.text().strip() == "" and self.codeEdit.text().strip( ) == "": QMessageBox.information(self, "提示", ' 科目名称和科目编码不能同时为空! ') return if self.subEdit.text().strip() != "" and self.codeEdit.text().strip( ) != "": QMessageBox.information(self, "提示", ' 科目名称和科目编码不能同时非空! ') return try: process_file(self.pathEdit.text().strip()) suffix = self.pathEdit.text().split(".")[-1] # print(self.pathEdit.text()) page = process_param(self.pageEdit.text().strip()) if suffix.lower() == "doc" or suffix.lower( ) == "docx" or suffix.lower() == "pdf": bound_info = self.boundEdit.text().strip() print(bound_info) if bound_info != "": if bound_info.endswith(","): bound_info = bound_info.rstrip(",") # print(page) if len(page) > 0: # print((list(map(str, page)))) pdf = camelot.read_pdf(self.pathEdit.text().strip(), flavor='stream', pages=','.join( list(map(str, page))), table_areas=[bound_info]) print(len(pdf)) for i in range(len(pdf)): table_list = [] for row_data in pdf[i].df.values.tolist(): table_list.append(row_data) self.parse(table_list) QMessageBox.information(self, "提示", ' 提取信息结束! ') print(self.json) else: """ pdf = pdfplumber.open(self.pathEdit.text().strip()) # print(pdf.pages) # print(len(pdf.pages)) # print(page) for i, _page in enumerate(pdf.pages): if i + 1 in page: table_list = [] print(_page.extract_text()) for pdf_table in _page.extract_tables(): for row in pdf_table: table_list.append(row) print(row) print(table_list) self.parse(table_list) """ if len(page) > 0: # print((list(map(str, page)))) if suffix.lower() == "doc" or suffix.lower() == "docx": pdf = camelot.read_pdf( self.pathEdit.text().strip().replace( "." + suffix, ".pdf"), flavor='stream', pages=','.join(list(map(str, page)))) else: pdf = camelot.read_pdf( self.pathEdit.text().strip(), flavor='stream', pages=','.join(list(map(str, page)))) # print(len(pdf)) for i in range(len(pdf)): table_list = [] for row_data in pdf[i].df.values.tolist(): table_list.append(row_data) self.parse(table_list) QMessageBox.information(self, "提示", ' 提取信息结束! ') elif suffix.lower() == "xls" or suffix.lower() == "xlsx": wb = xlrd.open_workbook(self.pathEdit.text().strip()) sheet_names = wb.sheet_names() for i, sheet_name in enumerate(sheet_names): # print(sheet_name) if i + 1 in page: table = wb.sheet_by_index(i) table_list = [] print(table.nrows) for ii in range(table.nrows): # print(type(table.row_values(ii))) # print(table.row_values(ii)) table_list.append(table.row_values(ii)) print(table_list) self.parse(table_list) QMessageBox.information(self, "提示", ' 提取信息结束! ') except: QMessageBox.information(self, "提示", "提取信息失败请查看输入是否有误!")
from utils import process_file, ResultMeasurements def print_results(result_measurements: ResultMeasurements): print( f'Hottest day is {result_measurements.hottest_day.date} with temperature ' f'{result_measurements.hottest_day.temperature}') print( f'Coldest day is {result_measurements.coldest_day.date} with temperature ' f'{result_measurements.coldest_day.temperature}') print( f'Hottest month is {result_measurements.hottest_month.date.strftime("%Y-%m")} with temperature ' f'{result_measurements.hottest_month.temperature}') print( f'Coldest month is {result_measurements.coldest_month.date.strftime("%Y-%m")} with temperature ' f'{result_measurements.coldest_month.temperature}') if __name__ == '__main__': filename = '34300.01.01.2016.01.01.2017.1.0.0.ru.csv' print_results(process_file(filename))
def evaluate(actor_critic, ob_rms, env_name, act_func, seed, k): num_stack = 1 log_interval = 1 load_dir = './trained_models/ppo/' #sd = np.random.randint(1000, size=1) sd = [1] env = make_env(env_name, int(sd[0]), 0, './tmp/test/' + str(seed) + '_' + act_func + '_') env = DummyVecEnv([env]) #actor_critic, ob_rms = \ # torch.load(os.path.join(load_dir, 'ppo_'+env_name+'_'+act_func+ '_'+str(seed)+".pt")) if len(env.observation_space.shape) == 1: env = VecNormalize(env, ret=False) env.ob_rms = ob_rms # An ugly hack to remove updates def _obfilt(self, obs): if self.ob_rms: obs = np.clip((obs - self.ob_rms.mean) / np.sqrt(self.ob_rms.var + self.epsilon), -self.clipob, self.clipob) return obs else: return obs env._obfilt = types.MethodType(_obfilt, env) render_func = env.venv.envs[0].render else: render_func = env.envs[0].render obs_shape = env.observation_space.shape obs_shape = (obs_shape[0] * num_stack, *obs_shape[1:]) current_obs = torch.zeros(1, *obs_shape) states = torch.zeros(1, actor_critic.state_size) masks = torch.zeros(1, 1) if torch.cuda.is_available(): current_obs = current_obs.cuda() states = states.cuda() masks = masks.cuda() def update_current_obs(obs): shape_dim0 = env.observation_space.shape[0] obs = torch.from_numpy(obs).float() if num_stack > 1: current_obs[:, :-shape_dim0] = current_obs[:, shape_dim0:] current_obs[:, -shape_dim0:] = obs #render_func('human') obs = env.reset() update_current_obs(obs) if env_name.find('Bullet') > -1: import pybullet as p torsoId = -1 for i in range(p.getNumBodies()): if (p.getBodyInfo(i)[0].decode() == "torso"): torsoId = i for i in range(5000): value, action, _, states = actor_critic.act(Variable(current_obs, volatile=True), Variable(states, volatile=True), Variable(masks, volatile=True), deterministic=True) #if i % 1000 == 0: # print("STEP: ", i) states = states.data cpu_actions = action.data.squeeze(1).cpu().numpy() # Obser reward and next obs obs, reward, done, _ = env.step(cpu_actions) masks.fill_(0.0 if done else 1.0) if current_obs.dim() == 4: current_obs *= masks.unsqueeze(2).unsqueeze(2) else: current_obs *= masks update_current_obs(obs) if env_name.find('Bullet') > -1: if torsoId > -1: distance = 5 yaw = 0 humanPos, humanOrn = p.getBasePositionAndOrientation(torsoId) p.resetDebugVisualizerCamera(distance, yaw, -20, humanPos) #render_func('human') #print(dir(env.envs[0])) process_file(env_name, act_func, sd[0], seed, k)
def train(): print("Configuring TensorBoard and Saver...") # 配置 Tensorboard,重新训练时,请将tensorboard文件夹删除,不然图会覆盖 tensorboard_dir = 'tensorboard/textcnn' if not os.path.exists(tensorboard_dir): os.makedirs(tensorboard_dir) if not os.path.exists(save_dir): os.makedirs(save_dir) print("Loading training and validation data...") # 载入训练集与验证集 start_time = time.time() x_train, y_train = process_file(train_dir, word_to_id, cat_to_id, config.seq_length) x_val, y_val = process_file(val_dir, word_to_id, cat_to_id, config.seq_length) time_dif = get_time_dif(start_time) print("Time usage:", time_dif) # 创建session with model.graph.as_default(): tf.summary.scalar("loss", model.loss) tf.summary.scalar("accuracy", model.acc) merged_summary = tf.summary.merge_all() writer = tf.summary.FileWriter(tensorboard_dir) saver = tf.train.Saver() model.session.run(tf.global_variables_initializer()) writer.add_graph(model.graph) print('Training and evaluating...') start_time = time.time() total_batch = 0 # 总批次 best_acc_val = 0.0 # 最佳验证集准确率 last_improved = 0 # 记录上一次提升批次 require_improvement = 1000 # 如果超过1000轮未提升,提前结束训练 flag = False for epoch in range(config.num_epochs): print('Epoch:', epoch + 1) batch_train = batch_iter(x_train, y_train, config.batch_size) for x_batch, y_batch in batch_train: feed_dict = feed_data(x_batch, y_batch, config.dropout_keep_prob) if total_batch % config.save_per_batch == 0: # 每多少轮次将训练结果写入tensorboard scalar s = model.session.run(merged_summary, feed_dict=feed_dict) writer.add_summary(s, total_batch) if total_batch % config.print_per_batch == 0: # 每多少轮次输出在训练集和验证集上的性能 feed_dict[model.keep_prob] = 1.0 loss_train, acc_train = model.session.run( [model.loss, model.acc], feed_dict=feed_dict) loss_val, acc_val = evaluate(model.session, x_val, y_val) # todo if acc_val > best_acc_val: # 保存最好结果 best_acc_val = acc_val last_improved = total_batch saver.save(sess=model.session, save_path=save_path) improved_str = '*' else: improved_str = '' time_dif = get_time_dif(start_time) msg = 'Iter: {0:>6}, Train Loss: {1:>6.2}, Train Acc: {2:>7.2%},' \ + ' Val Loss: {3:>6.2}, Val Acc: {4:>7.2%}, Time: {5} {6}' print( msg.format(total_batch, loss_train, acc_train, loss_val, acc_val, time_dif, improved_str)) model.session.run(model.optim, feed_dict=feed_dict) # 运行优化 total_batch += 1 if total_batch - last_improved > require_improvement: # 验证集正确率长期不提升,提前结束训练 print("No optimization for a long time, auto-stopping...") flag = True break # 跳出循环 if flag: # 同上 break
def handle_tsm(self): rm_tree('AddOns/TradeSkillMaster/External/EmbeddedLibs/') process_file( 'AddOns/TradeSkillMaster/TradeSkillMaster.toc', lambda lines: [line for line in lines if 'EmbeddedLibs' not in line])
def handle_omnicc(self): process_file( 'AddOns/OmniCC/core/core.xml', lambda lines: [line for line in lines if 'libs' not in line])