def process_item(self): clsname = self.__class__.__name__.lower() item = self.item line = item.get_line() m = self.match(line) i = line.lower().find(clsname) assert i!=-1,`clsname, line` self.prefix = line[:i].rstrip() self.name = line[i:m.end()].lstrip()[len(clsname):].strip() line = line[m.end():].lstrip() args = [] if line.startswith('('): i = line.find(')') assert i!=-1,`line` line2 = item.apply_map(line[:i+1]) for a in line2[1:-1].split(','): a=a.strip() if not a: continue args.append(a) line = line[i+1:].lstrip() suffix = item.apply_map(line) self.bind, suffix = parse_bind(suffix, item) self.result = None if isinstance(self, Function): self.result, suffix = parse_result(suffix, item) if suffix: assert self.bind is None,`self.bind` self.bind, suffix = parse_result(suffix, item) if self.result is None: self.result = self.name assert not suffix,`suffix` self.args = args self.typedecl = None return BeginStatement.process_item(self)
def process_item(self): clsname = self.__class__.__name__.lower() item = self.item line = item.get_line() m = self.match(line) i = line.lower().find(clsname) assert i != -1, ` clsname, line ` self.prefix = line[:i].rstrip() self.name = line[i:m.end()].lstrip()[len(clsname):].strip() line = line[m.end():].lstrip() args = [] if line.startswith('('): i = line.find(')') assert i != -1, ` line ` line2 = item.apply_map(line[:i + 1]) for a in line2[1:-1].split(','): a = a.strip() if not a: continue args.append(a) line = line[i + 1:].lstrip() suffix = item.apply_map(line) self.bind, suffix = parse_bind(suffix, item) self.result = None if isinstance(self, Function): self.result, suffix = parse_result(suffix, item) if suffix: assert self.bind is None, ` self.bind ` self.bind, suffix = parse_result(suffix, item) if self.result is None: self.result = self.name assert not suffix, ` suffix ` self.args = args self.typedecl = None return BeginStatement.process_item(self)
def infer_process(exe, program, reader, fetch_vars, dataset): """ the function to execute the infer process :param exe: the fluid Executor :param program: the infer_program :param reader: data reader :return: the list of prediction result """ def input_check(data): if data[0]['words'].lod()[0][-1] == 0: return data[0]['words'] return None results = [] for data in reader(): crf_decode = input_check(data) if crf_decode: results += utils.parse_result(crf_decode, crf_decode, dataset) continue words, crf_decode = exe.run( program, fetch_list=fetch_vars, feed=data, return_numpy=False, use_program_cache=True, ) results += utils.parse_result(words, crf_decode, dataset) return results
def do_infer(args): # init executor if args.use_cuda: place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0'))) else: place = fluid.CPUPlace() # define network and reader ernie_config = ErnieConfig(args.ernie_config_path) ernie_config.print_config() infer_program = fluid.Program() with fluid.program_guard(infer_program, fluid.default_startup_program()): with fluid.unique_name.guard(): infer_ret = creator.create_ernie_model(args, ernie_config, is_prediction=False) infer_program = infer_program.clone(for_test=True) print(args.test_data) pyreader, reader = creator.create_pyreader( args, file_name=args.test_data, feed_list=infer_ret['feed_list'], mode="ernie", place=place, iterable=True, return_reader=True, for_test=True) exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) # load model if not args.init_checkpoint: raise ValueError( "args 'init_checkpoint' should be set if only doing test or infer!" ) utils.init_checkpoint(exe, args.init_checkpoint, infer_program) # create dict id2word_dict = dict([(str(word_id), word) for word, word_id in reader.vocab.items()]) id2label_dict = dict([(str(label_id), label) for label, label_id in reader.label_map.items()]) Dataset = namedtuple("Dataset", ["id2word_dict", "id2label_dict"]) dataset = Dataset(id2word_dict, id2label_dict) # make prediction for data in pyreader(): (words, crf_decode) = exe.run( infer_program, fetch_list=[infer_ret["words"], infer_ret["crf_decode"]], feed=data[0], return_numpy=False) # User should notice that words had been clipped if long than args.max_seq_len results = utils.parse_result(words, crf_decode, dataset) for sent, tags in results: result_list = [ '(%s, %s)' % (ch, tag) for ch, tag in zip(sent, tags) ] print(''.join(result_list))
def test_inference_model(model_dir, text_list, dataset): """ :param model_dir: model's dir :param text_list: a list of input text, which decode as unicode :param dataset: :return: """ # init executor if args.use_cuda: place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0'))) else: place = fluid.CPUPlace() exe = fluid.Executor(place) # transfer text data to input tensor lod = [] for text in text_list: lod.append( np.array(dataset.word_to_ids(text.strip())).astype(np.int64)) base_shape = [[len(c) for c in lod]] tensor_words = fluid.create_lod_tensor(lod, base_shape, place) # for empty input, output the same empty if (sum(base_shape[0]) == 0): crf_decode = [tensor_words] else: # load inference model inference_scope = fluid.core.Scope() with fluid.scope_guard(inference_scope): [inferencer, feed_target_names, fetch_targets] = fluid.io.load_inference_model( model_dir, exe, model_filename='model.pdmodel', params_filename='params.pdparams', ) assert feed_target_names[0] == "words" print("Load inference model from %s" % (model_dir)) # get lac result crf_decode = exe.run( inferencer, feed={feed_target_names[0]: tensor_words}, fetch_list=fetch_targets, return_numpy=False, use_program_cache=True, ) # parse the crf_decode result result = utils.parse_result(tensor_words, crf_decode[0], dataset) for i, (sent, tags) in enumerate(result): result_list = ['(%s, %s)' % (ch, tag) for ch, tag in zip(sent, tags)] print(''.join(result_list))
def execute_selection(sql, values=None): """Runs a select query and returns the result :param sql: sql query :param value: values for the query :returns: query result """ with closing(mysql.connector.connect(**connetion_params)) as db: with closing(db.cursor(dictionary=True, buffered=True)) as cursor: if values == None: cursor.execute(sql) else: cursor.execute(sql, [values]) data = parse_result(cursor) return data
def main(args): """ Main Function """ args = parser.parse_args() ernie_config = ErnieConfig(args.ernie_config_path) ernie_config.print_config() if args.use_cuda: place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0'))) dev_count = fluid.core.get_cuda_device_count() else: place = fluid.CPUPlace() dev_count = int(os.environ.get('CPU_NUM', multiprocessing.cpu_count())) exe = fluid.Executor(place) reader = task_reader.SequenceLabelReader( vocab_path=args.vocab_path, label_map_config=args.label_map_config, max_seq_len=args.max_seq_len, do_lower_case=args.do_lower_case, in_tokens=False, random_seed=args.random_seed) if not (args.do_train or args.do_test or args.do_infer): raise ValueError("For args `do_train`, `do_val` and `do_test`, at " "least one of them must be True.") startup_prog = fluid.Program() if args.random_seed is not None: startup_prog.random_seed = args.random_seed if args.do_train: num_train_examples = reader.get_num_examples(args.train_set) max_train_steps = args.epoch * num_train_examples // args.batch_size // dev_count print("Device count: %d" % dev_count) print("Num train examples: %d" % num_train_examples) print("Max train steps: %d" % max_train_steps) train_program = fluid.Program() with fluid.program_guard(train_program, startup_prog): with fluid.unique_name.guard(): # create ernie_pyreader train_pyreader, ernie_inputs, words, labels = ernie_pyreader( args, pyreader_name='train_reader') train_pyreader.decorate_tensor_provider( reader.data_generator(args.train_set, args.batch_size, args.epoch, shuffle=True, phase="train")) # get ernie_embeddings embeddings = ernie_encoder(ernie_inputs, ernie_config=ernie_config) # user defined model based on ernie embeddings train_ret = create_model(args, embeddings, labels=labels, is_prediction=False) optimizer = fluid.optimizer.Adam(learning_rate=args.lr) fluid.clip.set_gradient_clip( clip=fluid.clip.GradientClipByGlobalNorm(clip_norm=1.0)) optimizer.minimize(train_ret["loss"]) lower_mem, upper_mem, unit = fluid.contrib.memory_usage( program=train_program, batch_size=args.batch_size) print("Theoretical memory usage in training: %.3f - %.3f %s" % (lower_mem, upper_mem, unit)) if args.do_test: test_program = fluid.Program() with fluid.program_guard(test_program, startup_prog): with fluid.unique_name.guard(): # create ernie_pyreader test_pyreader, ernie_inputs, words, labels = ernie_pyreader( args, pyreader_name='test_reader') test_pyreader.decorate_tensor_provider( reader.data_generator(args.test_set, args.batch_size, phase='test', epoch=1, shuffle=False)) # get ernie_embeddings embeddings = ernie_encoder(ernie_inputs, ernie_config=ernie_config) # user defined model based on ernie embeddings test_ret = create_model(args, embeddings, labels=labels, is_prediction=False) test_program = test_program.clone(for_test=True) if args.do_infer: infer_program = fluid.Program() with fluid.program_guard(infer_program, startup_prog): with fluid.unique_name.guard(): # create ernie_pyreader infer_pyreader, ernie_inputs, words, labels = ernie_pyreader( args, pyreader_name='infer_reader') infer_pyreader.decorate_tensor_provider( reader.data_generator(args.infer_set, args.batch_size, phase='infer', epoch=1, shuffle=False)) # get ernie_embeddings embeddings = ernie_encoder(ernie_inputs, ernie_config=ernie_config) # user defined model based on ernie embeddings infer_ret = create_model(args, embeddings, labels=labels, is_prediction=True) infer_ret["words"] = words infer_program = infer_program.clone(for_test=True) exe.run(startup_prog) # load checkpoints if args.do_train: if args.init_checkpoint and args.init_pretraining_params: print( "WARNING: args 'init_checkpoint' and 'init_pretraining_params' " "both are set! Only arg 'init_checkpoint' is made valid.") if args.init_checkpoint: utils.init_checkpoint(exe, args.init_checkpoint, startup_prog) elif args.init_pretraining_params: utils.init_pretraining_params(exe, args.init_pretraining_params, startup_prog) elif args.do_test or args.do_infer: if not args.init_checkpoint: raise ValueError( "args 'init_checkpoint' should be set if only doing test or infer!" ) utils.init_checkpoint(exe, args.init_checkpoint, startup_prog) if args.do_train: train_pyreader.start() steps = 0 total_cost, total_acc, total_num_seqs = [], [], [] while True: try: steps += 1 if steps % args.skip_steps == 0: fetch_list = [ train_ret["loss"], train_ret["num_infer_chunks"], train_ret["num_label_chunks"], train_ret["num_correct_chunks"], ] else: fetch_list = [] start_time = time.time() outputs = exe.run(program=train_program, fetch_list=fetch_list) end_time = time.time() if steps % args.skip_steps == 0: loss, nums_infer, nums_label, nums_correct = outputs train_ret["chunk_evaluator"].reset() train_ret["chunk_evaluator"].update( nums_infer, nums_label, nums_correct) precision, recall, f1_score = train_ret[ "chunk_evaluator"].eval() print( "[train] batch_id = %d, loss = %.5f, P: %.5f, R: %.5f, F1: %.5f, elapsed time %.5f, " "pyreader queue_size: %d " % (steps, loss, precision, recall, f1_score, end_time - start_time, train_pyreader.queue.size())) if steps % args.save_steps == 0: save_path = os.path.join(args.checkpoints, "step_" + str(steps)) print("\tsaving model as %s" % (save_path)) fluid.io.save_persistables(exe, save_path, train_program) if steps % args.validation_steps == 0: # evaluate test set if args.do_test: evaluate(exe, test_program, test_pyreader, test_ret) except fluid.core.EOFException: save_path = os.path.join(args.checkpoints, "step_" + str(steps)) fluid.io.save_persistables(exe, save_path, train_program) train_pyreader.reset() break # final eval on test set if args.do_test: evaluate(exe, test_program, test_pyreader, test_ret) if args.do_infer: # create dict id2word_dict = dict([(str(word_id), word) for word, word_id in reader.vocab.items()]) id2label_dict = dict([(str(label_id), label) for label, label_id in reader.label_map.items()]) Dataset = namedtuple("Dataset", ["id2word_dict", "id2label_dict"]) dataset = Dataset(id2word_dict, id2label_dict) infer_pyreader.start() while True: try: (words, crf_decode) = exe.run( infer_program, fetch_list=[infer_ret["words"], infer_ret["crf_decode"]], return_numpy=False) # User should notice that words had been clipped if long than args.max_seq_len results = utils.parse_result(words, crf_decode, dataset) for result in results: print(result) except fluid.core.EOFException: infer_pyreader.reset() break
lambda r: r.stats['SimulationTimeInSeconds']), ('Throughput', lambda r: r.stats['Throughput']), ('Average Packet Delay', lambda r: r.stats['AveragePacketDelay']), ('Payload Throughput', lambda r: r.stats['PayloadThroughput']), ('Average Payload Packet Delay', lambda r: r.stats['AveragePayloadPacketDelay']), ]) for data_packet_injection_rate in synthesized_data_packet_injection_rate_range: results = [] for traffic in synthesized_traffic_range: results.append( parse_result(working_directory(traffic, '', max_cycles, num_nodes, 'XY', 'Random', data_packet_injection_rate, -1, -1), bench=traffic)) results.append( parse_result(working_directory(traffic, '', max_cycles, num_nodes, 'OddEven', 'BufferLevel', data_packet_injection_rate, -1, -1), bench=traffic)) for aco_selection_alpha in aco_selection_alpha_range: for reinforcement_factor in reinforcement_factor_range: results.append( parse_result(working_directory(traffic, '', max_cycles, num_nodes, 'OddEven', 'ACO', data_packet_injection_rate, aco_selection_alpha, reinforcement_factor),
def main(args): startup_program = fluid.Program() if args.random_seed is not None: startup_program.random_seed = args.random_seed # prepare dataset dataset = reader.Dataset(args) if args.do_train: train_program = fluid.Program() if args.random_seed is not None: train_program.random_seed = args.random_seed with fluid.program_guard(train_program, startup_program): with fluid.unique_name.guard(): train_ret = create_model(args, "train_reader", dataset.vocab_size, dataset.num_labels) train_ret["pyreader"].decorate_paddle_reader( paddle.batch(paddle.reader.shuffle( dataset.file_reader(args.train_data), buf_size=args.traindata_shuffle_buffer), batch_size=args.batch_size)) optimizer = fluid.optimizer.Adam( learning_rate=args.base_learning_rate) optimizer.minimize(train_ret["avg_cost"]) if args.do_test: test_program = fluid.Program() with fluid.program_guard(test_program, startup_program): with fluid.unique_name.guard(): test_ret = create_model(args, "test_reader", dataset.vocab_size, dataset.num_labels) test_ret["pyreader"].decorate_paddle_reader( paddle.batch(dataset.file_reader(args.test_data), batch_size=args.batch_size)) test_program = test_program.clone( for_test=True) # to share parameters with train model if args.do_infer: infer_program = fluid.Program() with fluid.program_guard(infer_program, startup_program): with fluid.unique_name.guard(): infer_ret = create_model(args, "infer_reader", dataset.vocab_size, dataset.num_labels) infer_ret["pyreader"].decorate_paddle_reader( paddle.batch(dataset.file_reader(args.infer_data), batch_size=args.batch_size)) infer_program = infer_program.clone(for_test=True) # init executor if args.use_cuda: place = fluid.CUDAPlace(int(os.getenv('FLAGS_selected_gpus', '0'))) dev_count = fluid.core.get_cuda_device_count() else: place = fluid.CPUPlace() dev_count = multiprocessing.cpu_count() exe = fluid.Executor(place) exe.run(startup_program) # load checkpoints if args.do_train: if args.init_checkpoint: utils.init_checkpoint(exe, args.init_checkpoint, train_program) elif args.do_test: if not args.init_checkpoint: raise ValueError( "args 'init_checkpoint' should be set if only doing validation or testing!" ) utils.init_checkpoint(exe, args.init_checkpoint, test_program) if args.do_infer: utils.init_checkpoint(exe, args.init_checkpoint, infer_program) # do start to train if args.do_train: num_train_examples = dataset.get_num_examples(args.train_data) max_train_steps = args.epoch * num_train_examples // args.batch_size print("Num train examples: %d" % num_train_examples) print("Max train steps: %d" % max_train_steps) ce_info = [] batch_id = 0 for epoch_id in range(args.epoch): train_ret["pyreader"].start() ce_time = 0 try: while True: start_time = time.time() avg_cost, nums_infer, nums_label, nums_correct = exe.run( train_program, fetch_list=[ train_ret["avg_cost"], train_ret["num_infer_chunks"], train_ret["num_label_chunks"], train_ret["num_correct_chunks"], ], ) end_time = time.time() train_ret["chunk_evaluator"].reset() train_ret["chunk_evaluator"].update( nums_infer, nums_label, nums_correct) precision, recall, f1_score = train_ret[ "chunk_evaluator"].eval() batch_id += 1 print( "[train] batch_id = %d, loss = %.5f, P: %.5f, R: %.5f, F1: %.5f, elapsed time %.5f " % (batch_id, avg_cost, precision, recall, f1_score, end_time - start_time)) ce_time += end_time - start_time ce_info.append( [ce_time, avg_cost, precision, recall, f1_score]) # save checkpoints if (batch_id % args.save_model_per_batches == 0): save_path = os.path.join(args.model_save_dir, "step_" + str(batch_id)) fluid.io.save_persistables(exe, save_path, train_program) # evaluate if (batch_id % args.valid_model_per_batches == 0) and args.do_test: evaluate(exe, test_program, test_ret) except fluid.core.EOFException: save_path = os.path.join(args.model_save_dir, "step_" + str(batch_id)) fluid.io.save_persistables(exe, save_path, train_program) train_ret["pyreader"].reset() # break? if args.do_train and args.enable_ce: card_num = get_cards() ce_cost = 0 ce_f1 = 0 ce_p = 0 ce_r = 0 ce_time = 0 try: ce_time = ce_info[-2][0] ce_cost = ce_info[-2][1] ce_p = ce_info[-2][2] ce_r = ce_info[-2][3] ce_f1 = ce_info[-2][4] except: print("ce info error") print("kpis\teach_step_duration_card%s\t%s" % (card_num, ce_time)) print("kpis\ttrain_cost_card%s\t%f" % (card_num, ce_cost)) print("kpis\ttrain_precision_card%s\t%f" % (card_num, ce_p)) print("kpis\ttrain_recall_card%s\t%f" % (card_num, ce_r)) print("kpis\ttrain_f1_card%s\t%f" % (card_num, ce_f1)) # only test if args.do_test: evaluate(exe, test_program, test_ret) if args.do_infer: infer_ret["pyreader"].start() while True: try: ( words, crf_decode, ) = exe.run(infer_program, fetch_list=[ infer_ret["words"], infer_ret["crf_decode"], ], return_numpy=False) results = utils.parse_result(words, crf_decode, dataset) for result in results: print(result) except fluid.core.EOFException: infer_ret["pyreader"].reset() break
vocab, pred_useful, pred_verbs, pred_args, do_post=use_post_heuristics) pred_for_summarize[yid].append( get_ordered_pred_for_summarize(filtered_chunks)) else: srl_predictor = get_srl_predictor() dump_srl_raw_results = {} for yid in sents: dump_srl_raw_results[yid] = [] for sent in sents[yid]: srl = srl_predictor.predict_json({'sentence': sent}) srl_chunks = parse_result(srl) dump_srl_raw_results[yid].append((sent, srl_chunks)) srl_post_heuristics(srl_chunks, yid, vocab, pred_useful, pred_verbs, pred_args) print("Dumping SRL raw results") with open("raw_srl.pkl", 'wb') as raw_file: pickle.dump(dump_srl_raw_results, raw_file) print("Key sentence:") evaluate_keysent(gt, pred_useful) print("Verbs:") for sota_pred in (None, sota_pred_keysent): for fuzzy, partial_ratio in ((False, False), (True, False), (True, True)):