def plotFlopsData(flopsData, funcs, labelsToConnect, fileName, saveFolder): flopsFunc, accFunc, partitionFunc = funcs # create plots plots = [ FlopsStandardPlot(xFunc=flopsFunc, yFunc=accFunc), FlopsAveragePlot(flopsFunc, accFunc, labelsToConnect), # FlopsAveragePlot3D(partitionFunc, accFunc, labelsToConnect), FlopsMaxAccuracyPlot(flopsFunc, accFunc, labelsToConnect), MinFlopsPlot(flopsFunc, accFunc, labelsToConnect) ] # iterate 1st over non-integer keys for labelData in flopsData: for checkpoint in labelData.checkpoints(): for plot in plots: plot.addDataPoint(checkpoint) for plot in plots: plot.plot(labelData) # set plot properties for plot in plots: plot.setPlotProperties() # save as HTML Statistics.saveFigPDF([plot.fig for plot in plots], fileName, saveFolder)
def validate(self, valid_iter): """ Validate model. valid_iter: validate data iterator Returns: :obj:`nmt.Statistics`: validation loss statistics """ # Set model in validating mode. self.model.eval() stats = Statistics() with torch.no_grad(): for batch in valid_iter: src = make_features(batch, 'src') src = src.transpose(0, 1).contiguous() # _, src_lengths = batch.src src_lengths = (torch.ones(batch.batch_size) * src.size(1)).long() tgt = make_features(batch, 'tgt') # F-prop through the model. outputs, attns = self.model(src, tgt, src_lengths) # Compute loss. batch_stats = self.valid_loss.monolithic_compute_loss( batch, outputs, attns) # Update statistics. stats.update(batch_stats) # Set model back to training mode. self.model.train() return stats
def validate(self, valid_iter): """ Validate model. valid_iter: validate data iterator Returns: :obj:`nmt.Statistics`: validation loss statistics """ # Set model in validating mode. self.model.eval() stats = Statistics() for batch in valid_iter: src = make_features(batch, 'src') _, src_lengths = batch.src tgt = make_features(batch, 'tgt') # F-prop through the model. outputs, attns = self.model(src, tgt, src_lengths) # Compute loss. batch_stats = self.valid_loss.monolithic_compute_loss( batch, outputs, attns) # Update statistics. stats.update(batch_stats) # Set model back to training mode. self.model.train() return stats
def report_training(self, step, num_steps, learning_rate, report_stats, multigpu=False): """ This is the user-defined batch-level traing progress report function. Args: step(int): current step count. num_steps(int): total number of batches. learning_rate(float): current learning rate. report_stats(Statistics): old Statistics instance. Returns: report_stats(Statistics): updated Statistics instance. """ if self.start_time < 0: raise ValueError("""ReportMgr needs to be started (set 'start_time' or use 'start()'""") if step % self.report_every == 0: if multigpu: report_stats = \ Statistics.all_gather_stats(report_stats) self._report_training(step, num_steps, learning_rate, report_stats) self.progress_step += 1 return Statistics() else: return report_stats
def __init__(self, model_size, group_id, environment_id=0, training=True): self.model_size = model_size self._training = training self.environment_id = environment_id self.group_id = group_id # Build environment self.environment = Environment.create_environment(flags.env_type, self.environment_id, self._training) self.extrinsic_reward_manipulator = eval(flags.extrinsic_reward_manipulator) self.terminal = True self._composite_batch = CompositeBatch(maxlen=flags.replay_buffer_size if flags.replay_mean > 0 else 1) # Statistics self.__client_statistics = Statistics(flags.episode_count_for_evaluation) if self._training: #logs if not os.path.isdir(flags.log_dir + "/performance"): os.mkdir(flags.log_dir + "/performance") if not os.path.isdir(flags.log_dir + "/episodes"): os.mkdir(flags.log_dir + "/episodes") formatter = logging.Formatter('%(asctime)s %(message)s') # reward logger self.__reward_logger = logging.getLogger('reward_{}_{}'.format(self.group_id, self.environment_id)) hdlr = logging.FileHandler(flags.log_dir + '/performance/reward_{}_{}.log'.format(self.group_id, self.environment_id)) hdlr.setFormatter(formatter) self.__reward_logger.addHandler(hdlr) self.__reward_logger.setLevel(logging.DEBUG) self.__max_reward = float("-inf")
def setPlotProperties(self): Statistics.setPlotProperties(self.fig, self.ax, self.title, xLabel='', yLabel='', yMin=0.0, yMax=1.0)
def validate_one_epoch(val_loader, model, trainer, args): # switch to evaluate mode model.eval() val_stats = [Statistics() for i in range(5)] # read-in global entity list if args.dataset == 'kvr': with open('data/KVR/kvret_entities.json') as f: global_entity = json.load(f) global_entity_list = [] for key in global_entity.keys(): if key != 'poi': global_entity_list += [item.lower().replace(' ', '_') for item in global_entity[key]] else: for item in global_entity['poi']: global_entity_list += [item[k].lower().replace(' ', '_') for k in item.keys()] global_entity_list = list(set(global_entity_list)) else: raise NotImplementedError('Not implemented this val for datasets other than kvr yet.') with torch.no_grad(): end = time.time() # for i, data in tqdm(enumerate(val_loader)): cnt = 0 for data in tqdm(val_loader): # data = to_device() # end = time.time() cnt += 1 decoded_words = trainer.evaluate_batch(model, data) # logger.info("Decode Time cost: {}".format(str(time.time() - end))) # end = time.time() # update val states for each batch. val_stats = compute_val_stat(data, decoded_words, global_entity_list, val_stats, args) # logger.info("Val Compute Time cost: {}".format(str(time.time() - end))) if args.distributed: all_val_stats = Statistics.all_gather_stats_list(val_stats) else: all_val_stats = val_stats f1 = all_val_stats[0].accuracy() cal_f1 = all_val_stats[1].accuracy() wet_f1 = all_val_stats[2].accuracy() nav_f1 = all_val_stats[3].accuracy() logger.info("F1 SCORE:\t{}".format(str(f1))) logger.info("\tCAL F1:\t{}".format(str(cal_f1))) logger.info("\tWET F1:\t{}".format(str(wet_f1))) logger.info("\tNAV F1:\t{}".format(str(nav_f1))) bleu_score = all_val_stats[4].accuracy() / 100.0 # not validated yet. # bleu_score = 0.0 logger.info("\tBleu Score:\t{}".format(str(bleu_score))) return bleu_score, [f1, cal_f1, wet_f1, nav_f1]
def before_epoch(self, _epoch): cfg = self.cfg self._current_epoch = _epoch if cfg.n_gpu > 1: torch.distributed.barrier() self._model.train() self._epoch_tr_loss = 0.0 self._epoch_n_tr_steps = 0.0 if cfg.is_master_node: self._epoch_stats = Statistics(epoch_num=int(cfg.epoch_num), total_training_steps=self._optimizer.total_training_steps)
def test(self): result_file = '{}/test_results_{}.log'.format(flags.log_dir, self.global_step) if os.path.exists(result_file): print('Test results already produced and evaluated for {}'.format( result_file)) return result_lock = RLock() print('Start testing') testers = [] threads = [] tf_session = tf.get_default_session() tmp_environment = Environment.create_environment( env_type=flags.env_type, training=False) dataset_size = tmp_environment.get_dataset_size() data_per_thread = max(1, dataset_size // self.thread_count) for i in range(self.thread_count): # parallel testing tester = Group(group_id=-(i + 1), environment_count=data_per_thread, global_network=self.global_network, training=False) data_range_start = i * data_per_thread data_range_end = data_range_start + data_per_thread # print(data_range_start, data_per_thread, dataset_size) thread = Thread(target=self.test_function, args=(result_file, result_lock, tester, (data_range_start, data_range_end), tf_session)) thread.start() threads.append(thread) testers.append(tester) print('Test Set size:', dataset_size) print('Tests per thread:', data_per_thread) time.sleep(5) for thread in threads: # wait for all threads to end thread.join() print('End testing') # get overall statistics test_statistics = Statistics(self.thread_count) for group in testers: test_statistics.add(group.get_statistics()) info = test_statistics.get() # write results to file stats_file = '{}/test_statistics.log'.format(flags.log_dir) with open(stats_file, "a", encoding="utf-8") as file: # write stats to file file.write('{}\n'.format([ "{}={}".format(key, value) for key, value in sorted(info.items(), key=lambda t: t[0]) ])) print('Test statistics saved in {}'.format(stats_file)) print('Test results saved in {}'.format(result_file)) return tmp_environment.evaluate_test_results(result_file)
def main(): ITER_MAX = 30 list_statistics = [] e = Export(list_statistics) for i in range(1, 17): name_file = "" if i < 11: name_file = "./data/files/f{}.txt".format(i) else: name_file = "./data/files/Knapsack{}.txt".format(i - 10) statistics = Statistics(name_file, i, ITER_MAX) k = Knapsack(name_file) hcc = HillclimbingClassic() hcm = RandomSearch() vns = VNS(0) algorithms = [] algorithms.append(hcm) algorithms.append(hcc) algorithms.append(vns) hcm.max_efos = 1000 hcc.max_efos = 1000 vns.max_efos = 1000 information = [0] * 2 sublist_statistics = [0] * len(algorithms) print(name_file) j = 0 for algorithm in algorithms: vector = [] successfull_count = 0 start_time = time() for l in range(ITER_MAX): random.seed(l) k_max = random.randint( 2, int(math.log10(k.total_items) + 2)) if k.total_items < 6 else random.randint( 3, int(math.log10(k.total_items) + 3)) vns.k_max = k_max algorithm.execute(k, None) vector.append(algorithm.best_solution.fitness) successfull_count += 1 if algorithm.successfull else 0 end_time = time() information[0] = algorithm.__str__() information[1] = vector statistics.set_vector(information) statistics.successfull_count = successfull_count sublist_statistics[j] = copy.deepcopy(statistics) print("{}min\t\t{}".format(round((end_time - start_time) / 60, 3), algorithm)) j += 1 list_statistics.append(copy.deepcopy(sublist_statistics)) e.writeCSV() e.writeHTML()
def validate(test_loader, model, criterion, device): statistics = Statistics() # switch to evaluate mode model.eval() with torch.no_grad(): for batch_idx, (input_val, target_val) in enumerate(test_loader): loss, (prec1, prec5), y_pred, y_true = execute_batch( model, criterion, input_val, target_val, device) statistics.update(loss.data.cpu().numpy(), prec1, prec5, y_pred, y_true) return statistics
class ServerComponent(object): ''' ServerComponent ''' def __init__(self): self.statistics = Statistics() self.db_controller = database.DatabaseManager() self.queue_controller = None self.pool_controller = None self.is_started = False def start(self): main_logger.info('Starting server...') self.db_controller.add_categories() self.queue_controller = QueueController() self.pool_controller = PoolController( self.queue_controller.task_handler_queue, self.queue_controller.send_message_queue) self.pool_controller.start() self.is_started = True main_logger.info('Server succesfully started!') def stop(self): main_logger.info('Stopping server...') self.pool_controller.stop() self.is_started = False main_logger.info('Server succesfully stopped!') def run(self): try: while True: message = unicode(raw_input()) if message.startswith('quit'): self.stop() break elif message.startswith('stat'): print self.pool_controller.get_state() print self.queue_controller.get_state() print self.statistics.avg_time() elif message.startswith('restart'): main_logger.info('Restarting ServerComponent...') self.stop() self.start() main_logger.info('Server succesfully restarted!') else: main_logger.info('Wrong command') except KeyboardInterrupt: main_logger.info('Got KeyboardInterrupt... stopping server...') self.stop()
def sharded_compute_loss(self, batch, output, attns, cur_trunc, trunc_size, shard_size, normalization): """Compute the forward loss and backpropagate. Computation is done with shards and optionally truncation for memory efficiency. Also supports truncated BPTT for long sequences by taking a range in the decoder output sequence to back propagate in. Range is from `(cur_trunc, cur_trunc + trunc_size)`. Note sharding is an exact efficiency trick to relieve memory required for the generation buffers. Truncation is an approximate efficiency trick to relieve the memory required in the RNN buffers. Args: batch (batch) : batch of labeled examples output (:obj:`FloatTensor`) : output of decoder model `[tgt_len x batch x hidden]` attns (dict) : dictionary of attention distributions `[tgt_len x batch x src_len]` cur_trunc (int) : starting position of truncation window trunc_size (int) : length of truncation window shard_size (int) : maximum number of examples in a shard normalization (int) : Loss is divided by this number Returns: :obj:`onmt.utils.Statistics`: validation loss statistics """ batch_stats = Statistics() # 0-len_tgt_size range_ = (cur_trunc, cur_trunc + trunc_size) shard_state = self._make_shard_state(batch, output, range_, attns) ''' return { "output": output, "target": batch.tgt[1:len_tgt_size] } ''' # shard_size被设置为2 # shard: {"output": output_, "target": target_} for shard in shards(shard_state, shard_size): loss, stats = self._compute_loss(batch, **shard) # backward, 这里的div? loss.div(float(normalization)).backward() batch_stats.update(stats) return batch_stats
def get_sentiment_pie_charts( df_paragraphs: DataFrame, by_party: bool = True, parties: List[str] = None, media: List[str] = None ) -> List[Figure]: """ Get figures of pie charts for the sentiment either grouped by party or by media outlet. :param df_paragraphs: the dataframe of the paragraphs :param by_party: If True, group data by party, otherwise group by media :param parties: List of parties to consider. Defaults to all parties. :param media: List of media outlets to consider. Defaults to all media outlets. :return: List of figures containing the pie charts """ # Get sentiment statistics statistics = Statistics.get_sentiment_statistics(df_paragraphs, by_party, parties, media) # Define label and colors for pie charts labels = ["Positive", "Negative", "Neutral"] colors = ["#2ca02c", "#ff7f0e", "#1f77b4"] figures = [] for key_1, value_1 in statistics.items(): if by_party: # Group pie charts by party fig, axs = plt.subplots(1, len(value_1)) fig.suptitle("Sentiment towards {}".format(key_1)) if len(value_1) == 1: axs = [axs] figures.append(fig) else: # Group pie charts by media rows = 2 if len(value_1) > 2 else 1 columns = math.ceil(len(value_1) / 2) if len(value_1) > 2 else len(value_1) fig, axs = plt.subplots(rows, columns) fig.suptitle("Sentiment of {} towards parties".format(key_1)) if len(value_1) % 2 == 1 and len(value_1) > 1: fig.delaxes(axs.flatten()[-1]) if len(value_1) > 2: axs = [item for sublist in axs for item in sublist] if len(value_1) == 1: axs = [axs] figures.append(fig) # Create a pie chart plot for each item in the group for (key_2, value_2), ax in zip(value_1.items(), axs): ax.axis("equal") ax.set_title(key_2) ax.pie(value_2, colors=colors, counterclock=False, autopct="%1.1f%%", shadow=True, startangle=90) # Add legend to plot fig.legend(labels=labels, loc="lower right", borderaxespad=0.1, title="Sentiment") return figures
def __init__(self, cf): self.cf = cf self.net = None self.loss = None self.optimizer = None self.scheduler = None self.best_stats = Statistics()
def setPlotProperties(self): paddingPercentage = 0.02 paddingSize = (self.yMax - self.yMin) * paddingPercentage yMax = self.yMax + paddingSize yMin = self.yMin - paddingSize self.ax.locator_params(nbins=20, axis='y') from matplotlib.ticker import MultipleLocator spacing = 0.5 minorLocator = MultipleLocator(spacing) self.ax.yaxis.set_minor_locator(minorLocator) # Set grid to use minor tick locations. self.ax.grid(which='minor') Statistics.setPlotProperties(self.fig, self.ax, self.title, self.xLabel, self.yLabel, yMax, yMin)
def __init__(self, args: Namespace, logger: HtmlLogger): # init model model = self.buildModel(args) model = model.cuda() # create DataParallel model instance self.modelParallel = model # self.modelParallel = DataParallel(model, args.gpu) # assert (id(model) == id(self.modelParallel.module)) self.args = args self.model = model self.logger = logger # load data self.train_queue, self.valid_queue, self.createSearchQueue = load_data( args) # init train folder path, where to save loggers, checkpoints, etc. self.trainFolderPath = '{}/{}'.format(args.save, args.trainFolder) # build statistics containers containers = self.buildStatsContainers() # build statistics rules rules = self.buildStatsRules() # init statistics instance self.statistics = Statistics(containers, rules, args.save) # log parameters logParameters(logger, args, model)
def __init__(self, cf, model): self.cf = cf self.model = model self.logger_stats = Logger(cf.log_file_stats) self.stats = Statistics() self.msg = Messages() self.validator = self.validation(self.logger_stats, self.model, cf, self.stats, self.msg) self.trainer = self.train(self.logger_stats, self.model, cf, self.validator, self.stats, self.msg) self.predictor = self.predict(self.logger_stats, self.model, cf)
def get_basic_statistic_bar_plot(dataframe: DataFrame, parties: List[str], media: List[str]) -> Figure: """ Get the basic statistic (document distribution) figures. :param dataframe: Dataframe to extract the statistics from. :param parties: Parties to consider for the statistics. :param media: Media to consider for the statistics. :return: Figures containing the bar charts for each party. """ x, y = Statistics.get_basic_statistics(dataframe, media, parties) return Visualization._get_document_distribution_figure(x, y)
def __init__(self, group_id, model_id, environment_info, beta=None, training=True, parent=None, sibling=None): self.parameters_type = eval('tf.{}'.format(flags.parameters_type)) self.beta = beta if beta is not None else flags.beta self.value_count = 2 if flags.split_values else 1 # initialize self.training = training self.group_id = group_id self.model_id = model_id self.id = '{0}_{1}'.format(self.group_id,self.model_id) # model id self.parent = parent if parent is not None else self # used for sharing with other models in hierarchy, if any self.sibling = sibling if sibling is not None else self # used for sharing with other models in hierarchy, if any # Environment info action_shape = environment_info['action_shape'] self.policy_heads = [ { 'size':head[0], # number of actions to take 'depth':head[1] if len(head) > 1 else 0 # number of discrete action types: set 0 for continuous control } for head in action_shape ] state_shape = environment_info['state_shape'] self.state_heads = [ {'shape':head} for head in state_shape ] self.state_scaler = environment_info['state_scaler'] # state scaler, for saving memory (eg. in case of RGB input: uint8 takes less memory than float64) self.has_masked_actions = environment_info['has_masked_actions'] # Create the network self.build_input_placeholders() self.initialize_network() self.build_network() # Stuff for building the big-batch and optimize training computations self._big_batch_feed = [{},{}] self._batch_count = [0,0] self._train_batch_size = flags.batch_size*flags.big_batch_size # Statistics self._train_statistics = Statistics(flags.episode_count_for_evaluation) #======================================================================= # self.loss_distribution_estimator = RunningMeanStd(batch_size=flags.batch_size) #======================================================================= self.actor_loss_is_too_small = False
def validate(self, valid_iter, task_type='task'): """ Validate model. valid_iter: validate data iterator Returns: :obj:`nmt.Statistics`: validation loss statistics """ # Set model in validating mode. self.model.eval() stats = Statistics(task_type=task_type) with torch.no_grad(): for batch in valid_iter: src = make_features(batch, 'src') _, src_lengths = batch.src if task_type == 'task': tgt = make_features(batch, 'tgt') else: tgt = make_features(batch, 'tgt2') # F-prop through the model. outputs, attns = self.model(src, tgt, src_lengths, task_type=task_type) # Compute loss. if task_type == 'task': batch_stats = self.valid_loss.monolithic_compute_loss( batch, outputs, attns) else: batch_stats = self.valid_loss2.monolithic_compute_loss( batch, outputs, attns) # Update statistics. stats.update(batch_stats) # Set model back to training mode. self.model.train() return stats
def _report_training(self, step, num_steps, learning_rate, report_stats): """ See base class method `ReportMgrBase.report_training`. """ report_stats.output(step, num_steps, learning_rate, self.start_time) # Log the progress using the number of batches on the x-axis. self.maybe_log_tensorboard(report_stats, "progress", learning_rate, self.progress_step) report_stats = Statistics() return report_stats
class Writer: def __init__(self, opt): self.num_classes = opt.nclasses self.statistics = Statistics() def plot(self, epoch, phase, classes): #self.statistics.compute(self.num_classes) log_data(self.statistics, phase, classes, epoch) def plot_summary(self, phase, classes): #self.statistics.compute(self.num_classes) log_summary(self.statistics, phase, classes) def reset_counter(self): """ counts # of correct examples """ self.statistics = Statistics() def update_counter(self, loss, prec1, prec5, y_pred, y_true): self.statistics.update(loss, prec1, prec5, y_pred, y_true)
def _maybe_gather_stats(self, stat): """ Gather statistics in multi-processes cases Args: stat(:obj:onmt.utils.Statistics): a Statistics object to gather or None (it returns None in this case) Returns: stat: the updated (or unchanged) stat object """ if stat is not None and self.n_gpu > 1: return Statistics.all_gather_stats(stat) return stat
def train(self, train_steps, train_steps2, valid_steps): logger.info('Start training...') task_step = self.optim._task_step + 1 task2_step = self.optim._task2_step + 1 self.train_iter = self.get_task_batch(task_step, task_type='task') self.train_iter2 = self.get_task_batch(task2_step, task_type='task2') self.total_stats = Statistics(task_type='task') self.report_stats = Statistics(task_type='task') self._start_report_manager(self.report_manager, start_time=self.total_stats.start_time) self.total_stats2 = Statistics(task_type='task2') self.report_stats2 = Statistics(task_type='task2') self._start_report_manager(self.report_manager2, start_time=self.total_stats2.start_time) while task_step <= train_steps or task2_step <= train_steps2: self.save = False # self.save = True if task_step <= train_steps: task_step = self.train_task(task_step, train_steps, valid_steps, task_type='task') # self.save = False # self.save = True if task2_step <= train_steps2: task2_step = self.train_task(task2_step, train_steps2, valid_steps, task_type='task2') return self.total_stats, self.total_stats2
def _stats(self, loss, scores, target): """ Args: loss (:obj:`FloatTensor`): the loss computed by the loss criterion. scores (:obj:`FloatTensor`): a score for each possible output target (:obj:`FloatTensor`): true targets Returns: :obj:`onmt.utils.Statistics` : statistics for this batch. """ pred = scores.max(1)[1] non_padding = target.ne(self.padding_idx) num_correct = pred.eq(target).masked_select(non_padding).sum().item() num_non_padding = non_padding.sum().item() return Statistics(loss.item(), num_non_padding, num_correct)
def go(request): product_id = request.GET.get('id') try: obj = Product.objects.get(pk=product_id, is_active=True) except ObjectDoesNotExist: return HttpResponseNotFound() product_id = obj.id identify = get_request_identify(request) r = Statistics(product_id).add_uv( dict(meta=request.META, identify=identify)) if r == 'success': Product.objects.filter(pk=product_id).update(uv=F('uv') + 1) return HttpResponseRedirect(obj.url)
def get_party_statistics_bar_plots(dataframe: DataFrame, parties: List[str], media: List[str]) -> List[Figure]: """ Get the basic statistic (document distribution) figures of each party. :param dataframe: Dataframe to extract the statistics from. :param parties: Parties to consider for the statistics. :param media: Media to consider for the statistics. :return: Figures containing the bar charts for each party. """ figures = [] for party in parties: x, y = Statistics.get_party_statistics(dataframe, party, media) fig = Visualization._get_document_distribution_figure(x, y, party) figures.append(fig) return figures
def setUp(self): super().setUp() listeners = set() statistics = Statistics() self.source_server_clients = [] self.listener_server_clients = [] self.source_server = TornadoTCPServer(statistics, listeners, source.SourceHandler) sock, self.source_server_port = bind_unused_port() self.source_server.add_socket(sock) self.listener_server = TornadoTCPServer(statistics, listeners, listener.ListenerHandler) sock, self.listener_server_port = bind_unused_port() self.listener_server.add_socket(sock) self.source_client_msg_one = bytes( b'\x01' + int.to_bytes(1, 2, byteorder='big', signed=False) + 'abcdefgh'.encode() + b'\x01' + int.to_bytes(1, 1, byteorder='big', signed=False) + 'foofield'.encode() + int.to_bytes(1, 4, byteorder='big', signed=False)) self.source_client_msg_one += _get_xor(self.source_client_msg_one) self.source_client_msg_two = bytes( b'\x01' + int.to_bytes(1, 2, byteorder='big', signed=False) + 'ijklmnop'.encode() + b'\x01' + int.to_bytes(1, 1, byteorder='big', signed=False) + 'fieldfoo'.encode() + int.to_bytes(2, 4, byteorder='big', signed=False)) self.source_client_msg_two += _get_xor(self.source_client_msg_two) self.source_client_invalid_msg = bytes( b'\x01' + int.to_bytes(10, 2, byteorder='big', signed=False) + 'ijklmnop'.encode() + b'\x01' + int.to_bytes(1, 1, byteorder='big', signed=False) + 'fieldfoo'.encode() + int.to_bytes(2, 4, byteorder='big', signed=False) + int.to_bytes(1, 1, byteorder='big', signed=False))
def _stats(self, loss, scores, target): """ Args: loss (:obj:`FloatTensor`): the loss computed by the loss criterion. scores (:obj:`FloatTensor`): a score for each possible output target (:obj:`FloatTensor`): true targets 1 / x * vocab_size / x Returns: :obj:`onmt.utils.Statistics` : statistics for this batch. """ # 返回行的最大值的索引即预测的单词 # (shard_size*batch) pred = scores.max(1)[1] # (shard_size*batch) non_padding = target.ne(self.padding_idx) # 排除掉填充字符,得到正确的单词个数 num_correct = pred.eq(target).masked_select(non_padding).sum().item() # 总的单词个数 num_non_padding = non_padding.sum().item() return Statistics(loss.item(), num_non_padding, num_correct)
def sharded_compute_loss(self, batch, output, attns, shard_size, normalization, ratio): """Compute the forward loss and backpropagate. Computation is done with shards and optionally truncation for memory efficiency. Also supports truncated BPTT for long sequences by taking a range in the decoder output sequence to back propagate in. Range is from `(cur_trunc, cur_trunc + trunc_size)`. Note sharding is an exact efficiency trick to relieve memory required for the generation buffers. Truncation is an approximate efficiency trick to relieve the memory required in the RNN buffers. Args: batch (batch) : batch of labeled examples output (:obj:`FloatTensor`) : output of decoder model `[tgt_len x batch x hidden]` attns (dict) : dictionary of attention distributions `[tgt_len x batch x src_len]` cur_trunc (int) : starting position of truncation window trunc_size (int) : length of truncation window shard_size (int) : maximum number of examples in a shard normalization (int) : Loss is divided by this number Returns: :obj:`onmt.utils.Statistics`: validation loss statistics """ batch_stats = Statistics() shard_state = self._make_shard_state(batch, output, attns) for shard in shards({k: v[0] for k, v in shard_state.items()}, shard_size, retain_graph=True, ratio=ratio): loss, stats = self._compute_loss(batch, **shard) loss.div(float(normalization[0])).backward(retain_graph=True) batch_stats.update(stats) for shard in shards({k: v[1] for k, v in shard_state.items()}, shard_size, ratio=1 - ratio): loss, stats = self._compute_loss(batch, **shard) loss *= ratio loss.div(float(normalization[1])).backward() batch_stats.update(stats) return batch_stats
class TenhouClient(Client): statistics = None socket = None game_is_continue = True looking_for_game = True keep_alive_thread = None reconnected_messages = None decoder = TenhouDecoder() _count_of_empty_messages = 0 _rating_string = None _socket_mock = None def __init__(self, socket_mock=None): super().__init__() self.statistics = Statistics() self._socket_mock = socket_mock def connect(self): # for reproducer if self._socket_mock: self.socket = self._socket_mock TenhouClient._random_sleep = lambda x, y, z: 0 else: self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.connect((settings.TENHOU_HOST, settings.TENHOU_PORT)) def authenticate(self): self._send_message('<HELO name="{}" tid="f0" sx="M" />'.format(quote(settings.USER_ID))) messages = self._get_multiple_messages() auth_message = messages[0] if not auth_message: logger.info("Auth message wasn't received") return False # we reconnected to the game if '<GO' in auth_message: logger.info('Successfully reconnected') self.reconnected_messages = messages selected_game_type = self.decoder.parse_go_tag(auth_message) self._set_game_rules(selected_game_type) values = self.decoder.parse_names_and_ranks(messages[1]) self.table.set_players_names_and_ranks(values) return True auth_string, rating_string, new_rank_message = self.decoder.parse_hello_string(auth_message) self._rating_string = rating_string if not auth_string: logger.info("We didn't obtain auth string") return False if new_rank_message: logger.info('Achieved a new rank! \n {}'.format(new_rank_message)) auth_token = self.decoder.generate_auth_token(auth_string) self._send_message('<AUTH val="{}"/>'.format(auth_token)) self._send_message(self._pxr_tag()) # sometimes tenhou send an empty tag after authentication (in tournament mode) # and bot thinks that he was not auth # to prevent it lets wait a little bit # and lets read a group of tags continue_reading = True counter = 0 authenticated = False while continue_reading: messages = self._get_multiple_messages() for message in messages: if '<LN' in message: authenticated = True continue_reading = False counter += 1 # to avoid infinity loop if counter > 10: continue_reading = False if authenticated: self._send_keep_alive_ping() logger.info('Successfully authenticated') return True else: logger.info('Failed to authenticate') return False def start_game(self): log_link = '' # play in private or tournament lobby if settings.LOBBY != '0': if settings.IS_TOURNAMENT: logger.info('Go to the tournament lobby: {}'.format(settings.LOBBY)) self._send_message('<CS lobby="{}" />'.format(settings.LOBBY)) self._random_sleep(1, 2) self._send_message('<DATE />') else: logger.info('Go to the lobby: {}'.format(settings.LOBBY)) self._send_message('<CHAT text="{}" />'.format(quote('/lobby {}'.format(settings.LOBBY)))) self._random_sleep(1, 2) if self.reconnected_messages: # we already in the game self.looking_for_game = False self._send_message('<GOK />') self._random_sleep(1, 2) else: selected_game_type = self._build_game_type() game_type = '{},{}'.format(settings.LOBBY, selected_game_type) if not settings.IS_TOURNAMENT: self._send_message('<JOIN t="{}" />'.format(game_type)) logger.info('Looking for the game...') start_time = datetime.datetime.now() while self.looking_for_game: self._random_sleep(1, 2) messages = self._get_multiple_messages() for message in messages: if '<REJOIN' in message: # game wasn't found, continue to wait self._send_message('<JOIN t="{},r" />'.format(game_type)) if '<GO' in message: self._random_sleep(1, 2) self._send_message('<GOK />') self._send_message('<NEXTREADY />') # we had to have it there # because for tournaments we don't know # what exactly game type was set selected_game_type = self.decoder.parse_go_tag(message) process_rules = self._set_game_rules(selected_game_type) if not process_rules: logger.error('Hirosima (3 man) is not supported at the moment') self.end_game(success=False) return if '<TAIKYOKU' in message: self.looking_for_game = False game_id, seat = self.decoder.parse_log_link(message) log_link = 'http://tenhou.net/0/?log={}&tw={}'.format(game_id, seat) self.statistics.game_id = game_id if '<UN' in message: values = self.decoder.parse_names_and_ranks(message) self.table.set_players_names_and_ranks(values) self.statistics.username = values[0]['name'] if '<LN' in message: self._send_message(self._pxr_tag()) current_time = datetime.datetime.now() time_difference = current_time - start_time if time_difference.seconds > 60 * settings.WAITING_GAME_TIMEOUT_MINUTES: break # we wasn't able to find the game in specified time range # sometimes it happens and we need to end process # and try again later if self.looking_for_game: logger.error('Game is not started. Can\'t find the game') self.end_game() return logger.info('Game started') logger.info('Log: {}'.format(log_link)) logger.info('Players: {}'.format(self.table.players)) main_player = self.table.player meld_tile = None tile_to_discard = None while self.game_is_continue: self._random_sleep(1, 2) messages = self._get_multiple_messages() if self.reconnected_messages: messages = self.reconnected_messages + messages self.reconnected_messages = None if not messages: self._count_of_empty_messages += 1 else: # we had set to zero counter self._count_of_empty_messages = 0 for message in messages: if '<INIT' in message or '<REINIT' in message: values = self.decoder.parse_initial_values(message) self.table.init_round( values['round_wind_number'], values['count_of_honba_sticks'], values['count_of_riichi_sticks'], values['dora_indicator'], values['dealer'], values['scores'], ) logger.info('Round Log: {}&ts={}'.format(log_link, self.table.round_number)) tiles = self.decoder.parse_initial_hand(message) self.table.player.init_hand(tiles) logger.info(self.table) logger.info('Players: {}'.format(self.table.get_players_sorted_by_scores())) logger.info('Dealer: {}'.format(self.table.get_player(values['dealer']))) if '<REINIT' in message: players = self.decoder.parse_table_state_after_reconnection(message) for x in range(0, 4): player = players[x] for item in player['discards']: self.table.add_discarded_tile(x, item, False) for item in player['melds']: if x == 0: tiles = item.tiles main_player.tiles.extend(tiles) self.table.add_called_meld(x, item) # draw and discard if '<T' in message: win_suggestions = ['t="16"', 't="48"'] # we won by self draw (tsumo) if any(i in message for i in win_suggestions): self._random_sleep(1, 2) self._send_message('<N type="7" />') continue # Kyuushuu kyuuhai 「九種九牌」 # (9 kinds of honor or terminal tiles) if 't="64"' in message: self._random_sleep(1, 2) # TODO aim for kokushi self._send_message('<N type="9" />') continue drawn_tile = self.decoder.parse_tile(message) logger.info('Drawn tile: {}'.format(TilesConverter.to_one_line_string([drawn_tile]))) kan_type = self.player.should_call_kan(drawn_tile, False, main_player.in_riichi) if kan_type: self._random_sleep(1, 2) if kan_type == Meld.CHANKAN: meld_type = 5 logger.info('We upgraded pon to kan!') else: meld_type = 4 logger.info('We called a closed kan set!') self._send_message('<N type="{}" hai="{}" />'.format(meld_type, drawn_tile)) continue if not main_player.in_riichi: self.player.draw_tile(drawn_tile) discarded_tile = self.player.discard_tile() can_call_riichi = main_player.can_call_riichi() # let's call riichi if can_call_riichi: self._random_sleep(1, 2) self._send_message('<REACH hai="{}" />'.format(discarded_tile)) main_player.in_riichi = True else: # we had to add it to discards, to calculate remaining tiles correctly discarded_tile = drawn_tile self.table.add_discarded_tile(0, discarded_tile, True) # tenhou format: <D p="133" /> self._send_message('<D p="{}"/>'.format(discarded_tile)) logger.info('Discard: {}'.format(TilesConverter.to_one_line_string([discarded_tile]))) logger.info('Remaining tiles: {}'.format(self.table.count_of_remaining_tiles)) # new dora indicator after kan if '<DORA' in message: tile = self.decoder.parse_dora_indicator(message) self.table.add_dora_indicator(tile) logger.info('New dora indicator: {}'.format(TilesConverter.to_one_line_string([tile]))) if '<REACH' in message and 'step="1"' in message: who_called_riichi = self.decoder.parse_who_called_riichi(message) self.table.add_called_riichi(who_called_riichi) logger.info('Riichi called by {} player'.format(who_called_riichi)) # the end of round if '<AGARI' in message or '<RYUUKYOKU' in message: self._random_sleep(1, 2) self._send_message('<NEXTREADY />') # set was called if self.decoder.is_opened_set_message(message): meld = self.decoder.parse_meld(message) self.table.add_called_meld(meld.who, meld) logger.info('Meld: {} by {}'.format(meld, meld.who)) # tenhou confirmed that we called a meld # we had to do discard after this if meld.who == 0: if meld.type != Meld.KAN and meld.type != Meld.CHANKAN: discarded_tile = self.player.discard_tile(tile_to_discard) self.player.tiles.append(meld_tile) self._send_message('<D p="{}"/>'.format(discarded_tile)) win_suggestions = [ 't="8"', 't="9"', 't="10"', 't="11"', 't="12"', 't="13"', 't="15"' ] # we win by other player's discard if any(i in message for i in win_suggestions): # enemy called shouminkan and we can win there if self.decoder.is_opened_set_message(message): meld = self.decoder.parse_meld(message) tile = meld.called_tile enemy_seat = meld.who else: tile = self.decoder.parse_tile(message) enemy_seat = self.decoder.get_enemy_seat(message) self._random_sleep(1, 2) if main_player.should_call_win(tile, enemy_seat): self._send_message('<N type="6" />') else: self._send_message('<N />') if self.decoder.is_discarded_tile_message(message): tile = self.decoder.parse_tile(message) # <e21/> - is tsumogiri # <E21/> - discard from the hand if_tsumogiri = message[1].islower() player_seat = self.decoder.get_enemy_seat(message) # open hand suggestions if 't=' in message: # Possible t="" suggestions # 1 pon # 2 kan (it is a closed kan and can be send only to the self draw) # 3 pon + kan # 4 chi # 5 pon + chi # 7 pon + kan + chi # should we call a kan? if 't="3"' in message or 't="7"' in message: if self.player.should_call_kan(tile, True): self._random_sleep(1, 2) # 2 is open kan self._send_message('<N type="2" />') logger.info('We called an open kan set!') continue # player with "g" discard is always our kamicha is_kamicha_discard = False if message[1].lower() == 'g': is_kamicha_discard = True meld, tile_to_discard = self.player.try_to_call_meld(tile, is_kamicha_discard) if meld: self._random_sleep(1, 2) meld_tile = tile # 1 is pon meld_type = '1' if meld.type == Meld.CHI: # yeah it is 3, not 4 # because of tenhou protocol meld_type = '3' tiles = meld.tiles tiles.remove(meld_tile) # try to call a meld self._send_message('<N type="{}" hai0="{}" hai1="{}" />'.format( meld_type, tiles[0], tiles[1] )) # this meld will not improve our hand else: self._send_message('<N />') self.table.add_discarded_tile(player_seat, tile, if_tsumogiri) if 'owari' in message: values = self.decoder.parse_final_scores_and_uma(message) self.table.set_players_scores(values['scores'], values['uma']) if '<PROF' in message: self.game_is_continue = False # socket was closed by tenhou if self._count_of_empty_messages >= 5: logger.error('We are getting empty messages from socket. Probably socket connection was closed') self.end_game(False) return logger.info('Final results: {}'.format(self.table.get_players_sorted_by_scores())) # we need to finish the game, and only after this try to send statistics # if order will be different, tenhou will return 404 on log download endpoint self.end_game() # sometimes log is not available just after the game # let's wait one minute before the statistics update if settings.STAT_SERVER_URL: sleep(60) result = self.statistics.send_statistics() logger.info('Statistics sent: {}'.format(result)) def end_game(self, success=True): self.game_is_continue = False if success: self._send_message('<BYE />') if self.keep_alive_thread: self.keep_alive_thread.join() try: self.socket.shutdown(socket.SHUT_RDWR) self.socket.close() except OSError: pass if success: logger.info('End of the game') else: logger.error('Game was ended without success') def _send_message(self, message): # tenhou requires an empty byte in the end of each sending message logger.debug('Send: {}'.format(message)) message += '\0' self.socket.sendall(message.encode()) def _read_message(self): message = self.socket.recv(2048) logger.debug('Get: {}'.format(message.decode('utf-8').replace('\x00', ' '))) return message.decode('utf-8') def _get_multiple_messages(self): # tenhou can send multiple messages in one request messages = self._read_message() messages = messages.split('\x00') # last message always is empty after split, so let's exclude it messages = messages[0:-1] return messages def _send_keep_alive_ping(self): def send_request(): while self.game_is_continue: self._send_message('<Z />') # we can't use sleep(15), because we want to be able # end thread in the middle of running seconds_to_sleep = 15 for _ in range(0, seconds_to_sleep * 2): if self.game_is_continue: sleep(0.5) self.keep_alive_thread = Thread(target=send_request) self.keep_alive_thread.start() def _pxr_tag(self): # I have no idea why we need to send it, but better to do it if settings.IS_TOURNAMENT: return '<PXR V="-1" />' if settings.USER_ID == 'NoName': return '<PXR V="1" />' else: return '<PXR V="9" />' def _build_game_type(self): # usual case, we specified game type to play if settings.GAME_TYPE is not None: return settings.GAME_TYPE # kyu lobby, hanchan ari-ari default_game_type = '9' if settings.LOBBY != '0': logger.error("We can't use dynamic game type and custom lobby. Default game type was set") return default_game_type if not self._rating_string: logger.error("For NoName dynamic game type is not available. Default game type was set") return default_game_type temp = self._rating_string.split(',') dan = int(temp[0]) rate = float(temp[2]) logger.info('Player has {} rank and {} rate'.format(TenhouDecoder.RANKS[dan], rate)) game_type = default_game_type # dan lobby, we can play here from 1 kyu if dan >= 9: game_type = '137' # upperdan lobby, we can play here from 4 dan and with 1800+ rate if dan >= 13 and rate >= 1800: game_type = '41' # phoenix lobby, we can play here from 7 dan and with 2000+ rate if dan >= 16 and rate >= 2000: game_type = '169' return game_type def _set_game_rules(self, game_type): """ Set game related settings and return false, if we are trying to play 3 man game """ # need to find a better way to do it rules = bin(int(game_type)).replace('0b', '') while len(rules) != 8: rules = '0' + rules is_hanchan = rules[4] == '1' is_open_tanyao = rules[5] == '0' is_aka = rules[6] == '0' is_hirosima = rules[3] == '1' if is_hirosima: return False self.table.has_aka_dora = is_aka self.table.has_open_tanyao = is_open_tanyao logger.info('Game settings:') logger.info('Aka dora: {}'.format(self.table.has_aka_dora)) logger.info('Open tanyao: {}'.format(self.table.has_open_tanyao)) logger.info('Game type: {}'.format(is_hanchan and 'hanchan' or 'tonpusen')) return True def _random_sleep(self, min_sleep, max_sleep): sleep(random.randint(min_sleep, max_sleep + 1))
def __init__(self, socket_mock=None): super().__init__() self.statistics = Statistics() self._socket_mock = socket_mock
def __init__(self): self.statistics = Statistics() self.db_controller = database.DatabaseManager() self.queue_controller = None self.pool_controller = None self.is_started = False