def plotFlopsData(flopsData, funcs, labelsToConnect, fileName, saveFolder):
flopsFunc, accFunc, partitionFunc = funcs
# create plots
plots = [
FlopsStandardPlot(xFunc=flopsFunc, yFunc=accFunc),
FlopsAveragePlot(flopsFunc, accFunc, labelsToConnect),
# FlopsAveragePlot3D(partitionFunc, accFunc, labelsToConnect),
FlopsMaxAccuracyPlot(flopsFunc, accFunc, labelsToConnect),
MinFlopsPlot(flopsFunc, accFunc, labelsToConnect)
]
# iterate 1st over non-integer keys
for labelData in flopsData:
for checkpoint in labelData.checkpoints():
for plot in plots:
plot.addDataPoint(checkpoint)
for plot in plots:
plot.plot(labelData)
# set plot properties
for plot in plots:
plot.setPlotProperties()
# save as HTML
Statistics.saveFigPDF([plot.fig for plot in plots], fileName, saveFolder)
def validate(self, valid_iter):
""" Validate model.
valid_iter: validate data iterator
Returns:
:obj:`nmt.Statistics`: validation loss statistics
"""
# Set model in validating mode.
self.model.eval()
stats = Statistics()
with torch.no_grad():
for batch in valid_iter:
src = make_features(batch, 'src')
src = src.transpose(0, 1).contiguous()
# _, src_lengths = batch.src
src_lengths = (torch.ones(batch.batch_size) * src.size(1)).long()
tgt = make_features(batch, 'tgt')
# F-prop through the model.
outputs, attns = self.model(src, tgt, src_lengths)
# Compute loss.
batch_stats = self.valid_loss.monolithic_compute_loss(
batch, outputs, attns)
# Update statistics.
stats.update(batch_stats)
# Set model back to training mode.
self.model.train()
return stats
def validate(self, valid_iter):
""" Validate model.
valid_iter: validate data iterator
Returns:
:obj:`nmt.Statistics`: validation loss statistics
"""
# Set model in validating mode.
self.model.eval()
stats = Statistics()
for batch in valid_iter:
src = make_features(batch, 'src')
_, src_lengths = batch.src
tgt = make_features(batch, 'tgt')
# F-prop through the model.
outputs, attns = self.model(src, tgt, src_lengths)
# Compute loss.
batch_stats = self.valid_loss.monolithic_compute_loss(
batch, outputs, attns)
# Update statistics.
stats.update(batch_stats)
# Set model back to training mode.
self.model.train()
return stats
def report_training(self,
step,
num_steps,
learning_rate,
report_stats,
multigpu=False):
"""
This is the user-defined batch-level traing progress
report function.
Args:
step(int): current step count.
num_steps(int): total number of batches.
learning_rate(float): current learning rate.
report_stats(Statistics): old Statistics instance.
Returns:
report_stats(Statistics): updated Statistics instance.
"""
if self.start_time < 0:
raise ValueError("""ReportMgr needs to be started
(set 'start_time' or use 'start()'""")
if step % self.report_every == 0:
if multigpu:
report_stats = \
Statistics.all_gather_stats(report_stats)
self._report_training(step, num_steps, learning_rate, report_stats)
self.progress_step += 1
return Statistics()
else:
return report_stats
def __init__(self, model_size, group_id, environment_id=0, training=True):
self.model_size = model_size
self._training = training
self.environment_id = environment_id
self.group_id = group_id
# Build environment
self.environment = Environment.create_environment(flags.env_type, self.environment_id, self._training)
self.extrinsic_reward_manipulator = eval(flags.extrinsic_reward_manipulator)
self.terminal = True
self._composite_batch = CompositeBatch(maxlen=flags.replay_buffer_size if flags.replay_mean > 0 else 1)
# Statistics
self.__client_statistics = Statistics(flags.episode_count_for_evaluation)
if self._training:
#logs
if not os.path.isdir(flags.log_dir + "/performance"):
os.mkdir(flags.log_dir + "/performance")
if not os.path.isdir(flags.log_dir + "/episodes"):
os.mkdir(flags.log_dir + "/episodes")
formatter = logging.Formatter('%(asctime)s %(message)s')
# reward logger
self.__reward_logger = logging.getLogger('reward_{}_{}'.format(self.group_id, self.environment_id))
hdlr = logging.FileHandler(flags.log_dir + '/performance/reward_{}_{}.log'.format(self.group_id, self.environment_id))
hdlr.setFormatter(formatter)
self.__reward_logger.addHandler(hdlr)
self.__reward_logger.setLevel(logging.DEBUG)
self.__max_reward = float("-inf")
def setPlotProperties(self):
Statistics.setPlotProperties(self.fig,
self.ax,
self.title,
xLabel='',
yLabel='',
yMin=0.0,
yMax=1.0)
def validate_one_epoch(val_loader, model, trainer, args):
# switch to evaluate mode
model.eval()
val_stats = [Statistics() for i in range(5)]
# read-in global entity list
if args.dataset == 'kvr':
with open('data/KVR/kvret_entities.json') as f:
global_entity = json.load(f)
global_entity_list = []
for key in global_entity.keys():
if key != 'poi':
global_entity_list += [item.lower().replace(' ', '_') for item in global_entity[key]]
else:
for item in global_entity['poi']:
global_entity_list += [item[k].lower().replace(' ', '_') for k in item.keys()]
global_entity_list = list(set(global_entity_list))
else:
raise NotImplementedError('Not implemented this val for datasets other than kvr yet.')
with torch.no_grad():
end = time.time()
# for i, data in tqdm(enumerate(val_loader)):
cnt = 0
for data in tqdm(val_loader):
# data = to_device()
# end = time.time()
cnt += 1
decoded_words = trainer.evaluate_batch(model, data)
# logger.info("Decode Time cost: {}".format(str(time.time() - end)))
# end = time.time()
# update val states for each batch.
val_stats = compute_val_stat(data, decoded_words, global_entity_list, val_stats, args)
# logger.info("Val Compute Time cost: {}".format(str(time.time() - end)))
if args.distributed:
all_val_stats = Statistics.all_gather_stats_list(val_stats)
else:
all_val_stats = val_stats
f1 = all_val_stats[0].accuracy()
cal_f1 = all_val_stats[1].accuracy()
wet_f1 = all_val_stats[2].accuracy()
nav_f1 = all_val_stats[3].accuracy()
logger.info("F1 SCORE:\t{}".format(str(f1)))
logger.info("\tCAL F1:\t{}".format(str(cal_f1)))
logger.info("\tWET F1:\t{}".format(str(wet_f1)))
logger.info("\tNAV F1:\t{}".format(str(nav_f1)))
bleu_score = all_val_stats[4].accuracy() / 100.0
# not validated yet.
# bleu_score = 0.0
logger.info("\tBleu Score:\t{}".format(str(bleu_score)))
return bleu_score, [f1, cal_f1, wet_f1, nav_f1]
def before_epoch(self, _epoch):
cfg = self.cfg
self._current_epoch = _epoch
if cfg.n_gpu > 1:
torch.distributed.barrier()
self._model.train()
self._epoch_tr_loss = 0.0
self._epoch_n_tr_steps = 0.0
if cfg.is_master_node:
self._epoch_stats = Statistics(epoch_num=int(cfg.epoch_num),
total_training_steps=self._optimizer.total_training_steps)
def test(self):
result_file = '{}/test_results_{}.log'.format(flags.log_dir,
self.global_step)
if os.path.exists(result_file):
print('Test results already produced and evaluated for {}'.format(
result_file))
return
result_lock = RLock()
print('Start testing')
testers = []
threads = []
tf_session = tf.get_default_session()
tmp_environment = Environment.create_environment(
env_type=flags.env_type, training=False)
dataset_size = tmp_environment.get_dataset_size()
data_per_thread = max(1, dataset_size // self.thread_count)
for i in range(self.thread_count): # parallel testing
tester = Group(group_id=-(i + 1),
environment_count=data_per_thread,
global_network=self.global_network,
training=False)
data_range_start = i * data_per_thread
data_range_end = data_range_start + data_per_thread
# print(data_range_start, data_per_thread, dataset_size)
thread = Thread(target=self.test_function,
args=(result_file, result_lock, tester,
(data_range_start,
data_range_end), tf_session))
thread.start()
threads.append(thread)
testers.append(tester)
print('Test Set size:', dataset_size)
print('Tests per thread:', data_per_thread)
time.sleep(5)
for thread in threads: # wait for all threads to end
thread.join()
print('End testing')
# get overall statistics
test_statistics = Statistics(self.thread_count)
for group in testers:
test_statistics.add(group.get_statistics())
info = test_statistics.get()
# write results to file
stats_file = '{}/test_statistics.log'.format(flags.log_dir)
with open(stats_file, "a",
encoding="utf-8") as file: # write stats to file
file.write('{}\n'.format([
"{}={}".format(key, value)
for key, value in sorted(info.items(), key=lambda t: t[0])
]))
print('Test statistics saved in {}'.format(stats_file))
print('Test results saved in {}'.format(result_file))
return tmp_environment.evaluate_test_results(result_file)
def main():
ITER_MAX = 30
list_statistics = []
e = Export(list_statistics)
for i in range(1, 17):
name_file = ""
if i < 11:
name_file = "./data/files/f{}.txt".format(i)
else:
name_file = "./data/files/Knapsack{}.txt".format(i - 10)
statistics = Statistics(name_file, i, ITER_MAX)
k = Knapsack(name_file)
hcc = HillclimbingClassic()
hcm = RandomSearch()
vns = VNS(0)
algorithms = []
algorithms.append(hcm)
algorithms.append(hcc)
algorithms.append(vns)
hcm.max_efos = 1000
hcc.max_efos = 1000
vns.max_efos = 1000
information = [0] * 2
sublist_statistics = [0] * len(algorithms)
print(name_file)
j = 0
for algorithm in algorithms:
vector = []
successfull_count = 0
start_time = time()
for l in range(ITER_MAX):
random.seed(l)
k_max = random.randint(
2, int(math.log10(k.total_items) +
2)) if k.total_items < 6 else random.randint(
3, int(math.log10(k.total_items) + 3))
vns.k_max = k_max
algorithm.execute(k, None)
vector.append(algorithm.best_solution.fitness)
successfull_count += 1 if algorithm.successfull else 0
end_time = time()
information[0] = algorithm.__str__()
information[1] = vector
statistics.set_vector(information)
statistics.successfull_count = successfull_count
sublist_statistics[j] = copy.deepcopy(statistics)
print("{}min\t\t{}".format(round((end_time - start_time) / 60, 3),
algorithm))
j += 1
list_statistics.append(copy.deepcopy(sublist_statistics))
e.writeCSV()
e.writeHTML()
def validate(test_loader, model, criterion, device):
statistics = Statistics()
# switch to evaluate mode
model.eval()
with torch.no_grad():
for batch_idx, (input_val, target_val) in enumerate(test_loader):
loss, (prec1, prec5), y_pred, y_true = execute_batch(
model, criterion, input_val, target_val, device)
statistics.update(loss.data.cpu().numpy(), prec1, prec5, y_pred,
y_true)
return statistics
class ServerComponent(object):
'''
ServerComponent
'''
def __init__(self):
self.statistics = Statistics()
self.db_controller = database.DatabaseManager()
self.queue_controller = None
self.pool_controller = None
self.is_started = False
def start(self):
main_logger.info('Starting server...')
self.db_controller.add_categories()
self.queue_controller = QueueController()
self.pool_controller = PoolController(
self.queue_controller.task_handler_queue,
self.queue_controller.send_message_queue)
self.pool_controller.start()
self.is_started = True
main_logger.info('Server succesfully started!')
def stop(self):
main_logger.info('Stopping server...')
self.pool_controller.stop()
self.is_started = False
main_logger.info('Server succesfully stopped!')
def run(self):
try:
while True:
message = unicode(raw_input())
if message.startswith('quit'):
self.stop()
break
elif message.startswith('stat'):
print self.pool_controller.get_state()
print self.queue_controller.get_state()
print self.statistics.avg_time()
elif message.startswith('restart'):
main_logger.info('Restarting ServerComponent...')
self.stop()
self.start()
main_logger.info('Server succesfully restarted!')
else:
main_logger.info('Wrong command')
except KeyboardInterrupt:
main_logger.info('Got KeyboardInterrupt... stopping server...')
self.stop()
def sharded_compute_loss(self, batch, output, attns,
cur_trunc, trunc_size, shard_size,
normalization):
"""Compute the forward loss and backpropagate. Computation is done
with shards and optionally truncation for memory efficiency.
Also supports truncated BPTT for long sequences by taking a
range in the decoder output sequence to back propagate in.
Range is from `(cur_trunc, cur_trunc + trunc_size)`.
Note sharding is an exact efficiency trick to relieve memory
required for the generation buffers. Truncation is an
approximate efficiency trick to relieve the memory required
in the RNN buffers.
Args:
batch (batch) : batch of labeled examples
output (:obj:`FloatTensor`) :
output of decoder model `[tgt_len x batch x hidden]`
attns (dict) : dictionary of attention distributions
`[tgt_len x batch x src_len]`
cur_trunc (int) : starting position of truncation window
trunc_size (int) : length of truncation window
shard_size (int) : maximum number of examples in a shard
normalization (int) : Loss is divided by this number
Returns:
:obj:`onmt.utils.Statistics`: validation loss statistics
"""
batch_stats = Statistics()
# 0-len_tgt_size
range_ = (cur_trunc, cur_trunc + trunc_size)
shard_state = self._make_shard_state(batch, output, range_, attns)
'''
return {
"output": output,
"target": batch.tgt[1:len_tgt_size]
}
'''
# shard_size被设置为2
# shard: {"output": output_, "target": target_}
for shard in shards(shard_state, shard_size):
loss, stats = self._compute_loss(batch, **shard)
# backward, 这里的div?
loss.div(float(normalization)).backward()
batch_stats.update(stats)
return batch_stats
def get_sentiment_pie_charts(
df_paragraphs: DataFrame, by_party: bool = True, parties: List[str] = None, media: List[str] = None
) -> List[Figure]:
"""
Get figures of pie charts for the sentiment either grouped by party or by media outlet.
:param df_paragraphs: the dataframe of the paragraphs
:param by_party: If True, group data by party, otherwise group by media
:param parties: List of parties to consider. Defaults to all parties.
:param media: List of media outlets to consider. Defaults to all media outlets.
:return: List of figures containing the pie charts
"""
# Get sentiment statistics
statistics = Statistics.get_sentiment_statistics(df_paragraphs, by_party, parties, media)
# Define label and colors for pie charts
labels = ["Positive", "Negative", "Neutral"]
colors = ["#2ca02c", "#ff7f0e", "#1f77b4"]
figures = []
for key_1, value_1 in statistics.items():
if by_party:
# Group pie charts by party
fig, axs = plt.subplots(1, len(value_1))
fig.suptitle("Sentiment towards {}".format(key_1))
if len(value_1) == 1:
axs = [axs]
figures.append(fig)
else:
# Group pie charts by media
rows = 2 if len(value_1) > 2 else 1
columns = math.ceil(len(value_1) / 2) if len(value_1) > 2 else len(value_1)
fig, axs = plt.subplots(rows, columns)
fig.suptitle("Sentiment of {} towards parties".format(key_1))
if len(value_1) % 2 == 1 and len(value_1) > 1:
fig.delaxes(axs.flatten()[-1])
if len(value_1) > 2:
axs = [item for sublist in axs for item in sublist]
if len(value_1) == 1:
axs = [axs]
figures.append(fig)
# Create a pie chart plot for each item in the group
for (key_2, value_2), ax in zip(value_1.items(), axs):
ax.axis("equal")
ax.set_title(key_2)
ax.pie(value_2, colors=colors, counterclock=False, autopct="%1.1f%%", shadow=True, startangle=90)
# Add legend to plot
fig.legend(labels=labels, loc="lower right", borderaxespad=0.1, title="Sentiment")
return figures
def __init__(self, cf):
self.cf = cf
self.net = None
self.loss = None
self.optimizer = None
self.scheduler = None
self.best_stats = Statistics()
def setPlotProperties(self):
paddingPercentage = 0.02
paddingSize = (self.yMax - self.yMin) * paddingPercentage
yMax = self.yMax + paddingSize
yMin = self.yMin - paddingSize
self.ax.locator_params(nbins=20, axis='y')
from matplotlib.ticker import MultipleLocator
spacing = 0.5
minorLocator = MultipleLocator(spacing)
self.ax.yaxis.set_minor_locator(minorLocator)
# Set grid to use minor tick locations.
self.ax.grid(which='minor')
Statistics.setPlotProperties(self.fig, self.ax, self.title,
self.xLabel, self.yLabel, yMax, yMin)
def __init__(self, args: Namespace, logger: HtmlLogger):
# init model
model = self.buildModel(args)
model = model.cuda()
# create DataParallel model instance
self.modelParallel = model
# self.modelParallel = DataParallel(model, args.gpu)
# assert (id(model) == id(self.modelParallel.module))
self.args = args
self.model = model
self.logger = logger
# load data
self.train_queue, self.valid_queue, self.createSearchQueue = load_data(
args)
# init train folder path, where to save loggers, checkpoints, etc.
self.trainFolderPath = '{}/{}'.format(args.save, args.trainFolder)
# build statistics containers
containers = self.buildStatsContainers()
# build statistics rules
rules = self.buildStatsRules()
# init statistics instance
self.statistics = Statistics(containers, rules, args.save)
# log parameters
logParameters(logger, args, model)
def __init__(self, cf, model):
self.cf = cf
self.model = model
self.logger_stats = Logger(cf.log_file_stats)
self.stats = Statistics()
self.msg = Messages()
self.validator = self.validation(self.logger_stats, self.model, cf, self.stats, self.msg)
self.trainer = self.train(self.logger_stats, self.model, cf, self.validator, self.stats, self.msg)
self.predictor = self.predict(self.logger_stats, self.model, cf)
def get_basic_statistic_bar_plot(dataframe: DataFrame, parties: List[str], media: List[str]) -> Figure:
"""
Get the basic statistic (document distribution) figures.
:param dataframe: Dataframe to extract the statistics from.
:param parties: Parties to consider for the statistics.
:param media: Media to consider for the statistics.
:return: Figures containing the bar charts for each party.
"""
x, y = Statistics.get_basic_statistics(dataframe, media, parties)
return Visualization._get_document_distribution_figure(x, y)
def __init__(self, group_id, model_id, environment_info, beta=None, training=True, parent=None, sibling=None):
self.parameters_type = eval('tf.{}'.format(flags.parameters_type))
self.beta = beta if beta is not None else flags.beta
self.value_count = 2 if flags.split_values else 1
# initialize
self.training = training
self.group_id = group_id
self.model_id = model_id
self.id = '{0}_{1}'.format(self.group_id,self.model_id) # model id
self.parent = parent if parent is not None else self # used for sharing with other models in hierarchy, if any
self.sibling = sibling if sibling is not None else self # used for sharing with other models in hierarchy, if any
# Environment info
action_shape = environment_info['action_shape']
self.policy_heads = [
{
'size':head[0], # number of actions to take
'depth':head[1] if len(head) > 1 else 0 # number of discrete action types: set 0 for continuous control
}
for head in action_shape
]
state_shape = environment_info['state_shape']
self.state_heads = [
{'shape':head}
for head in state_shape
]
self.state_scaler = environment_info['state_scaler'] # state scaler, for saving memory (eg. in case of RGB input: uint8 takes less memory than float64)
self.has_masked_actions = environment_info['has_masked_actions']
# Create the network
self.build_input_placeholders()
self.initialize_network()
self.build_network()
# Stuff for building the big-batch and optimize training computations
self._big_batch_feed = [{},{}]
self._batch_count = [0,0]
self._train_batch_size = flags.batch_size*flags.big_batch_size
# Statistics
self._train_statistics = Statistics(flags.episode_count_for_evaluation)
#=======================================================================
# self.loss_distribution_estimator = RunningMeanStd(batch_size=flags.batch_size)
#=======================================================================
self.actor_loss_is_too_small = False
def validate(self, valid_iter, task_type='task'):
""" Validate model.
valid_iter: validate data iterator
Returns:
:obj:`nmt.Statistics`: validation loss statistics
"""
# Set model in validating mode.
self.model.eval()
stats = Statistics(task_type=task_type)
with torch.no_grad():
for batch in valid_iter:
src = make_features(batch, 'src')
_, src_lengths = batch.src
if task_type == 'task':
tgt = make_features(batch, 'tgt')
else:
tgt = make_features(batch, 'tgt2')
# F-prop through the model.
outputs, attns = self.model(src,
tgt,
src_lengths,
task_type=task_type)
# Compute loss.
if task_type == 'task':
batch_stats = self.valid_loss.monolithic_compute_loss(
batch, outputs, attns)
else:
batch_stats = self.valid_loss2.monolithic_compute_loss(
batch, outputs, attns)
# Update statistics.
stats.update(batch_stats)
# Set model back to training mode.
self.model.train()
return stats
def _report_training(self, step, num_steps, learning_rate, report_stats):
"""
See base class method `ReportMgrBase.report_training`.
"""
report_stats.output(step, num_steps, learning_rate, self.start_time)
# Log the progress using the number of batches on the x-axis.
self.maybe_log_tensorboard(report_stats, "progress", learning_rate,
self.progress_step)
report_stats = Statistics()
return report_stats
class Writer:
def __init__(self, opt):
self.num_classes = opt.nclasses
self.statistics = Statistics()
def plot(self, epoch, phase, classes):
#self.statistics.compute(self.num_classes)
log_data(self.statistics, phase, classes, epoch)
def plot_summary(self, phase, classes):
#self.statistics.compute(self.num_classes)
log_summary(self.statistics, phase, classes)
def reset_counter(self):
"""
counts # of correct examples
"""
self.statistics = Statistics()
def update_counter(self, loss, prec1, prec5, y_pred, y_true):
self.statistics.update(loss, prec1, prec5, y_pred, y_true)
def _maybe_gather_stats(self, stat):
"""
Gather statistics in multi-processes cases
Args:
stat(:obj:onmt.utils.Statistics): a Statistics object to gather
or None (it returns None in this case)
Returns:
stat: the updated (or unchanged) stat object
"""
if stat is not None and self.n_gpu > 1:
return Statistics.all_gather_stats(stat)
return stat
def train(self, train_steps, train_steps2, valid_steps):
logger.info('Start training...')
task_step = self.optim._task_step + 1
task2_step = self.optim._task2_step + 1
self.train_iter = self.get_task_batch(task_step, task_type='task')
self.train_iter2 = self.get_task_batch(task2_step, task_type='task2')
self.total_stats = Statistics(task_type='task')
self.report_stats = Statistics(task_type='task')
self._start_report_manager(self.report_manager,
start_time=self.total_stats.start_time)
self.total_stats2 = Statistics(task_type='task2')
self.report_stats2 = Statistics(task_type='task2')
self._start_report_manager(self.report_manager2,
start_time=self.total_stats2.start_time)
while task_step <= train_steps or task2_step <= train_steps2:
self.save = False
# self.save = True
if task_step <= train_steps:
task_step = self.train_task(task_step,
train_steps,
valid_steps,
task_type='task')
# self.save = False
# self.save = True
if task2_step <= train_steps2:
task2_step = self.train_task(task2_step,
train_steps2,
valid_steps,
task_type='task2')
return self.total_stats, self.total_stats2
def _stats(self, loss, scores, target):
"""
Args:
loss (:obj:`FloatTensor`): the loss computed by the loss criterion.
scores (:obj:`FloatTensor`): a score for each possible output
target (:obj:`FloatTensor`): true targets
Returns:
:obj:`onmt.utils.Statistics` : statistics for this batch.
"""
pred = scores.max(1)[1]
non_padding = target.ne(self.padding_idx)
num_correct = pred.eq(target).masked_select(non_padding).sum().item()
num_non_padding = non_padding.sum().item()
return Statistics(loss.item(), num_non_padding, num_correct)
def go(request):
product_id = request.GET.get('id')
try:
obj = Product.objects.get(pk=product_id, is_active=True)
except ObjectDoesNotExist:
return HttpResponseNotFound()
product_id = obj.id
identify = get_request_identify(request)
r = Statistics(product_id).add_uv(
dict(meta=request.META, identify=identify))
if r == 'success':
Product.objects.filter(pk=product_id).update(uv=F('uv') + 1)
return HttpResponseRedirect(obj.url)
def get_party_statistics_bar_plots(dataframe: DataFrame, parties: List[str], media: List[str]) -> List[Figure]:
"""
Get the basic statistic (document distribution) figures of each party.
:param dataframe: Dataframe to extract the statistics from.
:param parties: Parties to consider for the statistics.
:param media: Media to consider for the statistics.
:return: Figures containing the bar charts for each party.
"""
figures = []
for party in parties:
x, y = Statistics.get_party_statistics(dataframe, party, media)
fig = Visualization._get_document_distribution_figure(x, y, party)
figures.append(fig)
return figures
def setUp(self):
super().setUp()
listeners = set()
statistics = Statistics()
self.source_server_clients = []
self.listener_server_clients = []
self.source_server = TornadoTCPServer(statistics, listeners,
source.SourceHandler)
sock, self.source_server_port = bind_unused_port()
self.source_server.add_socket(sock)
self.listener_server = TornadoTCPServer(statistics, listeners,
listener.ListenerHandler)
sock, self.listener_server_port = bind_unused_port()
self.listener_server.add_socket(sock)
self.source_client_msg_one = bytes(
b'\x01' + int.to_bytes(1, 2, byteorder='big', signed=False) +
'abcdefgh'.encode() + b'\x01' +
int.to_bytes(1, 1, byteorder='big', signed=False) +
'foofield'.encode() +
int.to_bytes(1, 4, byteorder='big', signed=False))
self.source_client_msg_one += _get_xor(self.source_client_msg_one)
self.source_client_msg_two = bytes(
b'\x01' + int.to_bytes(1, 2, byteorder='big', signed=False) +
'ijklmnop'.encode() + b'\x01' +
int.to_bytes(1, 1, byteorder='big', signed=False) +
'fieldfoo'.encode() +
int.to_bytes(2, 4, byteorder='big', signed=False))
self.source_client_msg_two += _get_xor(self.source_client_msg_two)
self.source_client_invalid_msg = bytes(
b'\x01' + int.to_bytes(10, 2, byteorder='big', signed=False) +
'ijklmnop'.encode() + b'\x01' +
int.to_bytes(1, 1, byteorder='big', signed=False) +
'fieldfoo'.encode() +
int.to_bytes(2, 4, byteorder='big', signed=False) +
int.to_bytes(1, 1, byteorder='big', signed=False))
def _stats(self, loss, scores, target):
"""
Args:
loss (:obj:`FloatTensor`): the loss computed by the loss criterion.
scores (:obj:`FloatTensor`): a score for each possible output
target (:obj:`FloatTensor`): true targets
1 / x * vocab_size / x
Returns:
:obj:`onmt.utils.Statistics` : statistics for this batch.
"""
# 返回行的最大值的索引即预测的单词
# (shard_size*batch)
pred = scores.max(1)[1]
# (shard_size*batch)
non_padding = target.ne(self.padding_idx)
# 排除掉填充字符，得到正确的单词个数
num_correct = pred.eq(target).masked_select(non_padding).sum().item()
# 总的单词个数
num_non_padding = non_padding.sum().item()
return Statistics(loss.item(), num_non_padding, num_correct)
def sharded_compute_loss(self, batch, output, attns, shard_size,
normalization, ratio):
"""Compute the forward loss and backpropagate. Computation is done
with shards and optionally truncation for memory efficiency.
Also supports truncated BPTT for long sequences by taking a
range in the decoder output sequence to back propagate in.
Range is from `(cur_trunc, cur_trunc + trunc_size)`.
Note sharding is an exact efficiency trick to relieve memory
required for the generation buffers. Truncation is an
approximate efficiency trick to relieve the memory required
in the RNN buffers.
Args:
batch (batch) : batch of labeled examples
output (:obj:`FloatTensor`) :
output of decoder model `[tgt_len x batch x hidden]`
attns (dict) : dictionary of attention distributions
`[tgt_len x batch x src_len]`
cur_trunc (int) : starting position of truncation window
trunc_size (int) : length of truncation window
shard_size (int) : maximum number of examples in a shard
normalization (int) : Loss is divided by this number
Returns:
:obj:`onmt.utils.Statistics`: validation loss statistics
"""
batch_stats = Statistics()
shard_state = self._make_shard_state(batch, output, attns)
for shard in shards({k: v[0]
for k, v in shard_state.items()},
shard_size,
retain_graph=True,
ratio=ratio):
loss, stats = self._compute_loss(batch, **shard)
loss.div(float(normalization[0])).backward(retain_graph=True)
batch_stats.update(stats)
for shard in shards({k: v[1]
for k, v in shard_state.items()},
shard_size,
ratio=1 - ratio):
loss, stats = self._compute_loss(batch, **shard)
loss *= ratio
loss.div(float(normalization[1])).backward()
batch_stats.update(stats)
return batch_stats
class TenhouClient(Client):
statistics = None
socket = None
game_is_continue = True
looking_for_game = True
keep_alive_thread = None
reconnected_messages = None
decoder = TenhouDecoder()
_count_of_empty_messages = 0
_rating_string = None
_socket_mock = None
def __init__(self, socket_mock=None):
super().__init__()
self.statistics = Statistics()
self._socket_mock = socket_mock
def connect(self):
# for reproducer
if self._socket_mock:
self.socket = self._socket_mock
TenhouClient._random_sleep = lambda x, y, z: 0
else:
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.connect((settings.TENHOU_HOST, settings.TENHOU_PORT))
def authenticate(self):
self._send_message('<HELO name="{}" tid="f0" sx="M" />'.format(quote(settings.USER_ID)))
messages = self._get_multiple_messages()
auth_message = messages[0]
if not auth_message:
logger.info("Auth message wasn't received")
return False
# we reconnected to the game
if '<GO' in auth_message:
logger.info('Successfully reconnected')
self.reconnected_messages = messages
selected_game_type = self.decoder.parse_go_tag(auth_message)
self._set_game_rules(selected_game_type)
values = self.decoder.parse_names_and_ranks(messages[1])
self.table.set_players_names_and_ranks(values)
return True
auth_string, rating_string, new_rank_message = self.decoder.parse_hello_string(auth_message)
self._rating_string = rating_string
if not auth_string:
logger.info("We didn't obtain auth string")
return False
if new_rank_message:
logger.info('Achieved a new rank! \n {}'.format(new_rank_message))
auth_token = self.decoder.generate_auth_token(auth_string)
self._send_message('<AUTH val="{}"/>'.format(auth_token))
self._send_message(self._pxr_tag())
# sometimes tenhou send an empty tag after authentication (in tournament mode)
# and bot thinks that he was not auth
# to prevent it lets wait a little bit
# and lets read a group of tags
continue_reading = True
counter = 0
authenticated = False
while continue_reading:
messages = self._get_multiple_messages()
for message in messages:
if '<LN' in message:
authenticated = True
continue_reading = False
counter += 1
# to avoid infinity loop
if counter > 10:
continue_reading = False
if authenticated:
self._send_keep_alive_ping()
logger.info('Successfully authenticated')
return True
else:
logger.info('Failed to authenticate')
return False
def start_game(self):
log_link = ''
# play in private or tournament lobby
if settings.LOBBY != '0':
if settings.IS_TOURNAMENT:
logger.info('Go to the tournament lobby: {}'.format(settings.LOBBY))
self._send_message('<CS lobby="{}" />'.format(settings.LOBBY))
self._random_sleep(1, 2)
self._send_message('<DATE />')
else:
logger.info('Go to the lobby: {}'.format(settings.LOBBY))
self._send_message('<CHAT text="{}" />'.format(quote('/lobby {}'.format(settings.LOBBY))))
self._random_sleep(1, 2)
if self.reconnected_messages:
# we already in the game
self.looking_for_game = False
self._send_message('<GOK />')
self._random_sleep(1, 2)
else:
selected_game_type = self._build_game_type()
game_type = '{},{}'.format(settings.LOBBY, selected_game_type)
if not settings.IS_TOURNAMENT:
self._send_message('<JOIN t="{}" />'.format(game_type))
logger.info('Looking for the game...')
start_time = datetime.datetime.now()
while self.looking_for_game:
self._random_sleep(1, 2)
messages = self._get_multiple_messages()
for message in messages:
if '<REJOIN' in message:
# game wasn't found, continue to wait
self._send_message('<JOIN t="{},r" />'.format(game_type))
if '<GO' in message:
self._random_sleep(1, 2)
self._send_message('<GOK />')
self._send_message('<NEXTREADY />')
# we had to have it there
# because for tournaments we don't know
# what exactly game type was set
selected_game_type = self.decoder.parse_go_tag(message)
process_rules = self._set_game_rules(selected_game_type)
if not process_rules:
logger.error('Hirosima (3 man) is not supported at the moment')
self.end_game(success=False)
return
if '<TAIKYOKU' in message:
self.looking_for_game = False
game_id, seat = self.decoder.parse_log_link(message)
log_link = 'http://tenhou.net/0/?log={}&tw={}'.format(game_id, seat)
self.statistics.game_id = game_id
if '<UN' in message:
values = self.decoder.parse_names_and_ranks(message)
self.table.set_players_names_and_ranks(values)
self.statistics.username = values[0]['name']
if '<LN' in message:
self._send_message(self._pxr_tag())
current_time = datetime.datetime.now()
time_difference = current_time - start_time
if time_difference.seconds > 60 * settings.WAITING_GAME_TIMEOUT_MINUTES:
break
# we wasn't able to find the game in specified time range
# sometimes it happens and we need to end process
# and try again later
if self.looking_for_game:
logger.error('Game is not started. Can\'t find the game')
self.end_game()
return
logger.info('Game started')
logger.info('Log: {}'.format(log_link))
logger.info('Players: {}'.format(self.table.players))
main_player = self.table.player
meld_tile = None
tile_to_discard = None
while self.game_is_continue:
self._random_sleep(1, 2)
messages = self._get_multiple_messages()
if self.reconnected_messages:
messages = self.reconnected_messages + messages
self.reconnected_messages = None
if not messages:
self._count_of_empty_messages += 1
else:
# we had set to zero counter
self._count_of_empty_messages = 0
for message in messages:
if '<INIT' in message or '<REINIT' in message:
values = self.decoder.parse_initial_values(message)
self.table.init_round(
values['round_wind_number'],
values['count_of_honba_sticks'],
values['count_of_riichi_sticks'],
values['dora_indicator'],
values['dealer'],
values['scores'],
)
logger.info('Round Log: {}&ts={}'.format(log_link, self.table.round_number))
tiles = self.decoder.parse_initial_hand(message)
self.table.player.init_hand(tiles)
logger.info(self.table)
logger.info('Players: {}'.format(self.table.get_players_sorted_by_scores()))
logger.info('Dealer: {}'.format(self.table.get_player(values['dealer'])))
if '<REINIT' in message:
players = self.decoder.parse_table_state_after_reconnection(message)
for x in range(0, 4):
player = players[x]
for item in player['discards']:
self.table.add_discarded_tile(x, item, False)
for item in player['melds']:
if x == 0:
tiles = item.tiles
main_player.tiles.extend(tiles)
self.table.add_called_meld(x, item)
# draw and discard
if '<T' in message:
win_suggestions = ['t="16"', 't="48"']
# we won by self draw (tsumo)
if any(i in message for i in win_suggestions):
self._random_sleep(1, 2)
self._send_message('<N type="7" />')
continue
# Kyuushuu kyuuhai 「九種九牌」
# (9 kinds of honor or terminal tiles)
if 't="64"' in message:
self._random_sleep(1, 2)
# TODO aim for kokushi
self._send_message('<N type="9" />')
continue
drawn_tile = self.decoder.parse_tile(message)
logger.info('Drawn tile: {}'.format(TilesConverter.to_one_line_string([drawn_tile])))
kan_type = self.player.should_call_kan(drawn_tile, False, main_player.in_riichi)
if kan_type:
self._random_sleep(1, 2)
if kan_type == Meld.CHANKAN:
meld_type = 5
logger.info('We upgraded pon to kan!')
else:
meld_type = 4
logger.info('We called a closed kan set!')
self._send_message('<N type="{}" hai="{}" />'.format(meld_type, drawn_tile))
continue
if not main_player.in_riichi:
self.player.draw_tile(drawn_tile)
discarded_tile = self.player.discard_tile()
can_call_riichi = main_player.can_call_riichi()
# let's call riichi
if can_call_riichi:
self._random_sleep(1, 2)
self._send_message('<REACH hai="{}" />'.format(discarded_tile))
main_player.in_riichi = True
else:
# we had to add it to discards, to calculate remaining tiles correctly
discarded_tile = drawn_tile
self.table.add_discarded_tile(0, discarded_tile, True)
# tenhou format: <D p="133" />
self._send_message('<D p="{}"/>'.format(discarded_tile))
logger.info('Discard: {}'.format(TilesConverter.to_one_line_string([discarded_tile])))
logger.info('Remaining tiles: {}'.format(self.table.count_of_remaining_tiles))
# new dora indicator after kan
if '<DORA' in message:
tile = self.decoder.parse_dora_indicator(message)
self.table.add_dora_indicator(tile)
logger.info('New dora indicator: {}'.format(TilesConverter.to_one_line_string([tile])))
if '<REACH' in message and 'step="1"' in message:
who_called_riichi = self.decoder.parse_who_called_riichi(message)
self.table.add_called_riichi(who_called_riichi)
logger.info('Riichi called by {} player'.format(who_called_riichi))
# the end of round
if '<AGARI' in message or '<RYUUKYOKU' in message:
self._random_sleep(1, 2)
self._send_message('<NEXTREADY />')
# set was called
if self.decoder.is_opened_set_message(message):
meld = self.decoder.parse_meld(message)
self.table.add_called_meld(meld.who, meld)
logger.info('Meld: {} by {}'.format(meld, meld.who))
# tenhou confirmed that we called a meld
# we had to do discard after this
if meld.who == 0:
if meld.type != Meld.KAN and meld.type != Meld.CHANKAN:
discarded_tile = self.player.discard_tile(tile_to_discard)
self.player.tiles.append(meld_tile)
self._send_message('<D p="{}"/>'.format(discarded_tile))
win_suggestions = [
't="8"', 't="9"', 't="10"', 't="11"', 't="12"', 't="13"', 't="15"'
]
# we win by other player's discard
if any(i in message for i in win_suggestions):
# enemy called shouminkan and we can win there
if self.decoder.is_opened_set_message(message):
meld = self.decoder.parse_meld(message)
tile = meld.called_tile
enemy_seat = meld.who
else:
tile = self.decoder.parse_tile(message)
enemy_seat = self.decoder.get_enemy_seat(message)
self._random_sleep(1, 2)
if main_player.should_call_win(tile, enemy_seat):
self._send_message('<N type="6" />')
else:
self._send_message('<N />')
if self.decoder.is_discarded_tile_message(message):
tile = self.decoder.parse_tile(message)
# <e21/> - is tsumogiri
# <E21/> - discard from the hand
if_tsumogiri = message[1].islower()
player_seat = self.decoder.get_enemy_seat(message)
# open hand suggestions
if 't=' in message:
# Possible t="" suggestions
# 1 pon
# 2 kan (it is a closed kan and can be send only to the self draw)
# 3 pon + kan
# 4 chi
# 5 pon + chi
# 7 pon + kan + chi
# should we call a kan?
if 't="3"' in message or 't="7"' in message:
if self.player.should_call_kan(tile, True):
self._random_sleep(1, 2)
# 2 is open kan
self._send_message('<N type="2" />')
logger.info('We called an open kan set!')
continue
# player with "g" discard is always our kamicha
is_kamicha_discard = False
if message[1].lower() == 'g':
is_kamicha_discard = True
meld, tile_to_discard = self.player.try_to_call_meld(tile, is_kamicha_discard)
if meld:
self._random_sleep(1, 2)
meld_tile = tile
# 1 is pon
meld_type = '1'
if meld.type == Meld.CHI:
# yeah it is 3, not 4
# because of tenhou protocol
meld_type = '3'
tiles = meld.tiles
tiles.remove(meld_tile)
# try to call a meld
self._send_message('<N type="{}" hai0="{}" hai1="{}" />'.format(
meld_type,
tiles[0],
tiles[1]
))
# this meld will not improve our hand
else:
self._send_message('<N />')
self.table.add_discarded_tile(player_seat, tile, if_tsumogiri)
if 'owari' in message:
values = self.decoder.parse_final_scores_and_uma(message)
self.table.set_players_scores(values['scores'], values['uma'])
if '<PROF' in message:
self.game_is_continue = False
# socket was closed by tenhou
if self._count_of_empty_messages >= 5:
logger.error('We are getting empty messages from socket. Probably socket connection was closed')
self.end_game(False)
return
logger.info('Final results: {}'.format(self.table.get_players_sorted_by_scores()))
# we need to finish the game, and only after this try to send statistics
# if order will be different, tenhou will return 404 on log download endpoint
self.end_game()
# sometimes log is not available just after the game
# let's wait one minute before the statistics update
if settings.STAT_SERVER_URL:
sleep(60)
result = self.statistics.send_statistics()
logger.info('Statistics sent: {}'.format(result))
def end_game(self, success=True):
self.game_is_continue = False
if success:
self._send_message('<BYE />')
if self.keep_alive_thread:
self.keep_alive_thread.join()
try:
self.socket.shutdown(socket.SHUT_RDWR)
self.socket.close()
except OSError:
pass
if success:
logger.info('End of the game')
else:
logger.error('Game was ended without success')
def _send_message(self, message):
# tenhou requires an empty byte in the end of each sending message
logger.debug('Send: {}'.format(message))
message += '\0'
self.socket.sendall(message.encode())
def _read_message(self):
message = self.socket.recv(2048)
logger.debug('Get: {}'.format(message.decode('utf-8').replace('\x00', ' ')))
return message.decode('utf-8')
def _get_multiple_messages(self):
# tenhou can send multiple messages in one request
messages = self._read_message()
messages = messages.split('\x00')
# last message always is empty after split, so let's exclude it
messages = messages[0:-1]
return messages
def _send_keep_alive_ping(self):
def send_request():
while self.game_is_continue:
self._send_message('<Z />')
# we can't use sleep(15), because we want to be able
# end thread in the middle of running
seconds_to_sleep = 15
for _ in range(0, seconds_to_sleep * 2):
if self.game_is_continue:
sleep(0.5)
self.keep_alive_thread = Thread(target=send_request)
self.keep_alive_thread.start()
def _pxr_tag(self):
# I have no idea why we need to send it, but better to do it
if settings.IS_TOURNAMENT:
return '<PXR V="-1" />'
if settings.USER_ID == 'NoName':
return '<PXR V="1" />'
else:
return '<PXR V="9" />'
def _build_game_type(self):
# usual case, we specified game type to play
if settings.GAME_TYPE is not None:
return settings.GAME_TYPE
# kyu lobby, hanchan ari-ari
default_game_type = '9'
if settings.LOBBY != '0':
logger.error("We can't use dynamic game type and custom lobby. Default game type was set")
return default_game_type
if not self._rating_string:
logger.error("For NoName dynamic game type is not available. Default game type was set")
return default_game_type
temp = self._rating_string.split(',')
dan = int(temp[0])
rate = float(temp[2])
logger.info('Player has {} rank and {} rate'.format(TenhouDecoder.RANKS[dan], rate))
game_type = default_game_type
# dan lobby, we can play here from 1 kyu
if dan >= 9:
game_type = '137'
# upperdan lobby, we can play here from 4 dan and with 1800+ rate
if dan >= 13 and rate >= 1800:
game_type = '41'
# phoenix lobby, we can play here from 7 dan and with 2000+ rate
if dan >= 16 and rate >= 2000:
game_type = '169'
return game_type
def _set_game_rules(self, game_type):
"""
Set game related settings and
return false, if we are trying to play 3 man game
"""
# need to find a better way to do it
rules = bin(int(game_type)).replace('0b', '')
while len(rules) != 8:
rules = '0' + rules
is_hanchan = rules[4] == '1'
is_open_tanyao = rules[5] == '0'
is_aka = rules[6] == '0'
is_hirosima = rules[3] == '1'
if is_hirosima:
return False
self.table.has_aka_dora = is_aka
self.table.has_open_tanyao = is_open_tanyao
logger.info('Game settings:')
logger.info('Aka dora: {}'.format(self.table.has_aka_dora))
logger.info('Open tanyao: {}'.format(self.table.has_open_tanyao))
logger.info('Game type: {}'.format(is_hanchan and 'hanchan' or 'tonpusen'))
return True
def _random_sleep(self, min_sleep, max_sleep):
sleep(random.randint(min_sleep, max_sleep + 1))
def __init__(self, socket_mock=None):
super().__init__()
self.statistics = Statistics()
self._socket_mock = socket_mock
def __init__(self):
self.statistics = Statistics()
self.db_controller = database.DatabaseManager()
self.queue_controller = None
self.pool_controller = None
self.is_started = False
