def __init__(self,
fnames,
labels,
root_dir,
train=True,
mean=None,
std=None):
self.fnames = fnames
self.labels = labels
self.melspec_dir = root_dir
self.mean = mean
self.std = std
self.fnames = [
os.path.splitext(os.path.basename(fname))[0]
for fname in self.fnames
]
self.fnames = [
self.melspec_dir + '/' + fname + '.npy' for fname in self.fnames
]
self.transform = None
self.pil = transforms.ToPILImage()
if train:
self.transform = albumentations_transform
self._find_min_width()
if self.mean is None:
self.stats = Statistics()
self._update_stats()
self.mean = self.stats.mean()
self.std = self.stats.stddev()
def __init__(self, initial_states=None, use_stats=True):
self.predictor_states = initial_states
self.trgt_sentence = []
self.score, self.base_score = 0.0, 0.0
self.score_breakdown = []
self.word_to_consume = None
self.statistics = Statistics() if use_stats else None
class AggStats:
def __init__(self):
self.time_connect_stats = Statistics()
self.time_total_stats = Statistics()
self.n_results = 0
self.n_fatal_errors = 0
self.n_attempts = 0
self.n_429 = 0
self.n_errors = 0
def __str__(self):
return "conn:{:0.2f}s, resp:{:0.2f}s, throttle:{:.1%}, err:{}+{}({:.1%}) | {:.1%} success".format(
self.time_connect_stats.mean(),
self.time_total_stats.mean(),
self.throttle_ratio(),
self.n_errors - self.n_fatal_errors,
self.n_fatal_errors,
self.error_ratio(),
self.success_ratio(),
)
@zero_on_division_error
def throttle_ratio(self):
return self.n_429 / self.n_attempts
@zero_on_division_error
def error_ratio(self):
return self.n_errors / self.n_attempts
@zero_on_division_error
def success_ratio(self):
return self.n_results / (self.n_results + self.n_fatal_errors)
def __init__(self, actions, epochs=3, batch_size=128, update_num=256,
gamma_e=0.99, gamma_i=0.99, lambda_=.95, learning_rate=0.0003,
grad_clip=0.5, **kargs):
super(Trainer, self).__init__()
self.agent_ppo=ppo_intris_value(actions, **kargs)
self.agent_rnd=random_network_distilation(**kargs)
self.epochs=epochs
self.batch_size=batch_size
self.update_num=update_num
self.gamma_e=gamma_e
self.gamma_i=gamma_i
self.lambda_=lambda_
self.grad_clip=grad_clip
self.init_replay_buffer={
'states': [],
'next_states': [],
'logits': [],
'actions': [],
'values_e': [],
'values_i': [],
'rewards_e': [],
'rewards_i': [],
'rewards_i_raw': [],
'dones': [],
}
self.replay_buffer=deepcopy(self.init_replay_buffer)
self.running_stats = Statistics()
self.optimizer = tf.keras.optimizers.Adam(learning_rate, epsilon=1e-10)
def __init__(self):
self.time_connect_stats = Statistics()
self.time_total_stats = Statistics()
self.n_results = 0
self.n_fatal_errors = 0
self.n_attempts = 0
self.n_429 = 0
self.n_errors = 0
def __init__(self,
want_max=True,
want_mean=True,
want_stdev=True,
want_min=True):
self.stat = Statistics()
self.want_max = want_max
self.want_mean = want_mean
self.want_stdev = want_stdev
self.want_min = want_min
def add_timing(name: str, elapsed: float, no_print=True) -> Statistics:
global _timings
stats = _timings.get(name, None)
if stats is None:
stats = Statistics()
_timings[name] = stats
stats.push(elapsed)
if not no_print:
logging.info('Timing "{}": {}s'.format(name, elapsed))
return stats
class AggStats:
def __init__(self):
self.time_connect_stats = Statistics()
self.time_total_stats = Statistics()
self.n_results = 0
self.n_fatal_errors = 0
self.n_attempts = 0
self.n_429 = 0
self.n_errors = 0
self.n_input_queries = 0
self.n_extracted_queries = 0 # Queries answered without any type of error
self.n_query_responses = 0
self.n_billable_query_responses = 0 # Some errors are also billed
def __str__(self):
return "conn:{:0.2f}s, resp:{:0.2f}s, throttle:{:.1%}, err:{}+{}({:.1%}) | success:{}/{}({:.1%})".format(
self.time_connect_stats.mean(), self.time_total_stats.mean(),
self.throttle_ratio(),
self.n_errors - self.n_fatal_errors, self.n_fatal_errors,
self.error_ratio(), self.n_extracted_queries, self.n_input_queries,
self.success_ratio())
def summary(self):
return ("\n" + "Summary\n" + "-------\n" +
"Mean connection time: {:0.2f}\n".format(
self.time_connect_stats.mean()) +
"Mean response time: {:0.2f}\n".format(
self.time_total_stats.mean()) +
"Throttle ratio: {:0.1%}\n".format(
self.throttle_ratio()) +
"Attempts: {}\n".format(self.n_attempts) +
"Errors: {:0.1%}, fatal: {}, non fatal: {}\n"
.format(self.error_ratio(), self.n_fatal_errors,
self.n_errors - self.n_fatal_errors) +
"Successful URLs: {} of {}\n".format(
self.n_extracted_queries, self.n_input_queries) +
"Success ratio: {:0.1%}\n".format(
self.success_ratio()) +
"Billable query responses: {} of {}\n".format(
self.n_billable_query_responses, self.n_query_responses))
@zero_on_division_error
def throttle_ratio(self):
return self.n_429 / self.n_attempts
@zero_on_division_error
def error_ratio(self):
return self.n_errors / self.n_attempts
@zero_on_division_error
def success_ratio(self):
return self.n_extracted_queries / self.n_input_queries
def __init__(self):
try:
self.encoder = PKCS7Encoder()
self.runningStat = Statistics()
self.rAvg = 0.0
self.rDev = 0.0
if "AES_KEY" in os.environ:
self.skey = os.environ["AES_KEY"]
else:
self.skey = os.urandom(16)
self.iv = ''.join(
[chr(random.randint(0, 0xFF)) for i in range(16)])
except Exception as e:
print "Failed to Initialize NsuCryptoServer:" + str(e)
def __init__(self):
self.time_connect_stats = Statistics()
self.time_total_stats = Statistics()
self.n_results = 0
self.n_fatal_errors = 0
self.n_attempts = 0
self.n_429 = 0
self.n_errors = 0
self.n_input_queries = 0
self.n_extracted_queries = 0 # Queries answered without any type of error
self.n_query_responses = 0
self.n_billable_query_responses = 0 # Some errors are also billed
def __init__(self, name, task_runner, max_workers=2):
self.name = name
self.reader = None
self.writer = None
self.message_prefix = f"{application_edge_marker}{self.name}{serial_sep}"
self.stats = Statistics()
self.executor = ProcessPoolExecutor(max_workers=max_workers)
self._task_runner = task_runner
self.ack_cond = asyncio.Condition()
self.response_lock = asyncio.Lock()
self.was_cancelled = False
class EpochStats:
def __init__(self) -> None:
self.start_lr = Statistics()
self.end_lr = Statistics()
self.train_fold = FoldStats()
self.val_fold = FoldStats()
def update(self, epoch_node: OrderedDict) -> None:
self.start_lr.push(epoch_node['start_lr'])
if 'train' in epoch_node:
self.end_lr.push(epoch_node['train']['end_lr'])
self.train_fold.update(epoch_node['train'])
if 'val' in epoch_node:
self.val_fold.update(epoch_node['val'])
def main():
parser = argparse.ArgumentParser(description='NAS E2E Runs')
parser.add_argument('--logdir',
type=str,
default='D:\\logdir\\azure\\random_cifar_test',
help='folder with logs')
args, extra_args = parser.parse_known_args()
lines = []
top1s = []
for filepath in pathlib.Path(args.logdir).rglob('logs.log'):
epoch = 0
for line in pathlib.Path(filepath).read_text().splitlines():
if '[eval_test] Epoch: [ 1/1] ' in line:
top1s.append(Statistics())
top1 = float(line.strip().split('(')[-1].split(',')[0].split(
'%')[0].strip()) / 100.0
lines.append(f'{epoch}\t{top1}\t{str(filepath)}')
top1s[epoch].push(top1)
epoch += 1
pathlib.Path(os.path.join(args.logdir,
'summary.tsv')).write_text('\n'.join(lines))
stat_lines = ['epoch\tmean\tstddev\tcount']
for i, top1 in enumerate(top1s):
stat_lines.append(
f'{i}\t{top1.mean()}\t{top1.stddev() if len(top1)>1 else float("NaN")}\t{len(top1)}'
)
pathlib.Path(os.path.join(args.logdir, 'summary_stats.tsv')).write_text(
'\n'.join(stat_lines))
def __init__(self, metric_funcs):
self.metrics = {metric: Statistics() for metric in metric_funcs}
self.metrics_values = {
metric: [[] for _ in range(11)]
for metric in metric_funcs
}
self.metric_funcs = metric_funcs
def __init__(self, metric_funcs):
"""
Args:
metric_funcs (dict): A dict where the keys are metric names and the
values are Python functions for evaluating that metric.
"""
self.metrics = {metric: Statistics() for metric in metric_funcs}
def simulate(QUEUES, SERVERS):
x = []
y = []
# we want to test the system while changing lambda
for l in range(1, POINTS, 1):
lambd = 2.0 * SERVERS * QUEUES * mu * l / POINTS #service at most of arrival time
env = simpy.Environment()
# save stats
env.stats = Statistics()
# queues
env.queues = [Servers(env, SERVERS, mu) for i in range(0, QUEUES)]
# start the arrival process
env.process(arrival(env, lambd))
# simulate until SIM_TIME
env.run(until=SIM_TIME)
# print the mean response time
#print("%.3f %.3f %.2f" % (mu, lambd, env.stats.mean()))
x.append(lambd / (SERVERS * QUEUES * mu))
y.append(env.stats.mean())
return x, y
def make_stats_array_recursive(shape):
if len(shape) == 0:
return Statistics()
else:
return [
make_stats_array_recursive(shape[1:])
for _ in range(shape[0])
]
def __init__(self, metric_funcs, name=None):
self.metrics = {
metric: Statistics() for metric in metric_funcs
}
self.values = {
metric: [] for metric in metric_funcs
}
self.metric_funcs = metric_funcs
self.name = name
class ImageStats:
def __init__(self) -> None:
self.dims = set()
self.counts = {}
self.sizes = Statistics()
self.suffixes = set()
self.lock = Lock()
def push(self, filepath: pathlib.Path) -> None:
with Image.open(str(filepath)) as img:
shape = np.array(img).shape
filesize = filepath.stat().st_size
with self.lock:
self.dims.add(shape)
parent = str(filepath.parent)
self.counts[parent] = self.counts.get(parent, 0) + 1
self.sizes.push(filesize)
self.suffixes.add(filepath.suffix)
def __init__(self):
# variables of our model: the queue of "clients"
self.waiting_line = Queue()
# whether the server is idle or not
self.idle = True
# Statistics of the waiting time
self.response_time = Statistics()
def __init__(self, env: GoalEnv, skill_reset_steps: int):
super().__init__(env)
self._skill_reset_steps = skill_reset_steps
self._skill_dim = env.observation_space["desired_goal"].shape[0]
obs_dim = self.env.observation_space["observation"].shape[0]
self.observation_space = gym.spaces.Box(-np.inf, np.inf, shape=(obs_dim + self._skill_dim, ))
self.strategy = MVNStrategy(skill_dim=self._skill_dim)
self._cur_skill = self.strategy.sample_skill()
self._last_dict_obs = None
self._goal_deltas_stats = [Statistics([1e-6]) for _ in range(self._skill_dim)]
def plot_confidence_curve(self, save_dir):
""" Creates a plot of confidence vs. each metric. """
confidences = np.array(self.confidences)
confidence_order = np.flip(confidences.argsort())
for metric, values in self.metrics.items():
stats = Statistics()
cum_metrics = []
for ex_idx in confidence_order:
stats.push(values[ex_idx])
cum_metrics.append(stats.mean())
plt.plot(cum_metrics)
plt.xlabel('Confidence Ranking')
plt.ylabel(metric)
save_pickle({
'confidences': confidences,
'metrics': cum_metrics
}, os.path.join(save_dir, '{}.p'.format(metric)))
plt.savefig(os.path.join(save_dir, '{}.png'.format(metric)))
plt.clf()
def __init__(self, metric_funcs, output_path, method):
"""
Parameters
----------
metric_funcs (dict): A dict where the keys are metric names and the values are Python functions for evaluating
that metric.
output_path: path to the output directory
method: reconstruction method
"""
self.metrics_scores = {metric: Statistics() for metric in metric_funcs}
self.output_path = output_path
self.method = method
class FoldStats:
def __init__(self) -> None:
self.top1 = Statistics()
self.top5 = Statistics()
self.duration = Statistics()
self.step_time = Statistics()
def update(self, fold_node: OrderedDict) -> None:
self.top1.push(fold_node['top1'])
self.top5.push(fold_node['top5'])
if 'duration' in fold_node:
self.duration.push(fold_node['duration'])
if 'step_time' in fold_node:
self.step_time.push(fold_node['step_time'])
class Event:
def __init__(self,
want_max=True,
want_mean=True,
want_stdev=True,
want_min=True):
self.stat = Statistics()
self.want_max = want_max
self.want_mean = want_mean
self.want_stdev = want_stdev
self.want_min = want_min
def to_kvhf(self):
mins = [self.stat.minimum()] if self.want_min else []
maxs = [self.stat.maximum()] if self.want_max else []
#This is necessary cause bug of runstat https://github.com/grantjenks/python-runstats/issues/27
if len(self.stat) > 1:
stdevs = [self.stat.stddev()] if self.want_stdev else []
else:
stdevs = [0]
means = [self.stat.mean()] if self.want_mean else []
return Serie_stats(means=means, mins=mins, maxs=maxs, stdevs=stdevs)
def add_occurence(self, time):
self.stat.push(time)
def get_occurence_num(self):
return len(self.stat)
def test_pushandrecalculate(self):
rstat = Statistics()
resp = self.app.get('/api/reset')
self.assertEqual(resp.status_code, 200)
test_array = [4, 7, 6, 9, 1]
for i in test_array:
resp = self.app.post('/api/pushandrecalculate', data=str(i))
print resp.get_data(as_text=True)
avg = 0
for i in test_array:
avg = avg + i
avg = float(avg) / float(len(test_array))
self.assertEqual(
resp.get_data(
as_text=True).split("{")[1].split(",")[0].encode('ascii'),
str(avg))
for i in test_array:
rstat.push(i)
self.assertEqual(
str(rstat.stddev(ddof=0)),
resp.get_data(
as_text=True).split("}")[0].split(",")[-1].encode('ascii'))
print "--- Push and Recalculate Statistics Validation Successful ---"
def sstdComp(self, data, id):
if self.outl is None and self.score is None:
self.outl = []
self.score = []
else:
self.outl.clear()
self.score.clear()
if id not in self.stats:
self.stats[id] = Statistics()
for i in data[:]:
self.stats[id].push(i)
if self.stats[id].get_state()[0] > 1.0:
sigma = self.stats[id].mean() + self.sigma*self.stats[id].stddev()
for i in range(0, len(data[:])):
if data[i] >= sigma:
self.outl.append(-1)
self.score.append(abs(data[i] - self.stats[id].stddev()))
else:
self.outl.append(1)
self.score.append(abs(data[i] - self.stats[id].stddev()))
def get_summary_text(log_key:str, out_dir:str, node_path:str, epoch_stats:List[EpochStats], seed_runs:int)->str:
lines = ['','']
lines.append(f'## Run: {log_key}\n')
lines.append(f'### Metric Type: {node_path}\n')
lines.append(f'Number of epochs: {len(epoch_stats)}\n')
lines.append(f'Number of seeds: {seed_runs}\n')
lines.append('\n')
plot_filename = get_valid_filename(log_key + ':' + node_path)+'.png'
plot_filepath = os.path.join(out_dir, plot_filename)
plot_epochs(epoch_stats, plot_filepath)
lines.append('')
train_duration = Statistics()
for epoch_stat in epoch_stats:
train_duration += epoch_stat.train_fold.duration
lines.append(f'')
lines.append(f'Train epoch time: {stat2str(train_duration)}')
lines.append('')
milestones = [0, 5, 30, 100, 200, 600, 1500]
for milestone in milestones:
if len(epoch_stats) >= milestone and len(epoch_stats[milestone-1].val_fold.top1)>0:
lines.append(f'{stat2str(epoch_stats[milestone-1].val_fold.top1)} val top1 @ {milestone} epochs\n')
# last epoch
if not len(epoch_stats) in milestones:
# find last epoch with valid stats
last_epoch = len(epoch_stats)-1
while last_epoch>=0 and len(epoch_stats[last_epoch].val_fold.top1)==0:
last_epoch -= 1
if last_epoch >=0:
lines.append(f'{stat2str(epoch_stats[last_epoch].val_fold.top1)} val top1 @ {len(epoch_stats)} epochs [Last]\n')
else:
lines.append(f'[Last] No epoch with valid val stats found!')
return '\n'.join(lines)
def simulate(QUEUES, SERVERS):
y = []
for i in range(0, SAMPLES):
env = simpy.Environment()
# save stats
env.stats = Statistics()
# queues
env.queues = [Servers(env, SERVERS, mu) for i in range(0, QUEUES)]
# start the arrival process
env.process(arrival(env, lambd))
# simulate until SIM_TIME
env.run(until=SIM_TIME)
y.append(env.stats.mean())
return y
def __init__(self,
env: GoalEnv,
skill_reset_steps: int = -1,
skill_dim=None):
super().__init__(env)
self._env_is_ant = hasattr(env, "IS_ANT") and env.IS_ANT
self._do_reset_skill = skill_reset_steps > -1
self._skill_reset_steps = skill_reset_steps
self._skill_dim = skill_dim or env.observation_space[
"desired_goal"].shape[0]
obs_dim = self.env.observation_space["observation"].shape[0]
self.observation_space = gym.spaces.Box(-np.inf,
np.inf,
shape=(obs_dim +
self._skill_dim, ))
self.strategy = MVNStrategy(skill_dim=self._skill_dim)
self._cur_skill = self.strategy.sample_skill()
self._last_dict_obs = None
self._goal_deltas_stats = [
Statistics([1e-6]) for _ in range(self._skill_dim)
]
self._latest_goal_delta_stats = dict()
def test_statistics():
alpha = [random.random() for val in range(count)]
alpha_stats = Statistics()
for val in alpha:
alpha_stats.push(val)
assert len(alpha_stats) == count
assert error(mean(alpha), alpha_stats.mean()) < error_limit
assert error(variance(alpha), alpha_stats.variance()) < error_limit
assert error(stddev(alpha), alpha_stats.stddev()) < error_limit
assert error(skewness(alpha), alpha_stats.skewness()) < error_limit
assert error(kurtosis(alpha), alpha_stats.kurtosis()) < error_limit
assert alpha_stats.minimum() == min(alpha)
assert alpha_stats.maximum() == max(alpha)
alpha_stats.clear()
assert len(alpha_stats) == 0
alpha_stats = Statistics(alpha)
beta = [random.random() for val in range(count)]
beta_stats = Statistics()
for val in beta:
beta_stats.push(val)
gamma_stats = alpha_stats + beta_stats
assert len(beta_stats) != len(gamma_stats)
assert error(mean(alpha + beta), gamma_stats.mean()) < error_limit
assert error(variance(alpha + beta), gamma_stats.variance()) < error_limit
assert error(stddev(alpha + beta), gamma_stats.stddev()) < error_limit
assert error(skewness(alpha + beta), gamma_stats.skewness()) < error_limit
assert error(kurtosis(alpha + beta), gamma_stats.kurtosis()) < error_limit
assert gamma_stats.minimum() == min(alpha + beta)
assert gamma_stats.maximum() == max(alpha + beta)
delta_stats = beta_stats.copy()
delta_stats += alpha_stats
assert len(beta_stats) != len(delta_stats)
assert error(mean(alpha + beta), delta_stats.mean()) < error_limit
assert error(variance(alpha + beta), delta_stats.variance()) < error_limit
assert error(stddev(alpha + beta), delta_stats.stddev()) < error_limit
assert error(skewness(alpha + beta), delta_stats.skewness()) < error_limit
assert error(kurtosis(alpha + beta), delta_stats.kurtosis()) < error_limit
assert delta_stats.minimum() == min(alpha + beta)
assert delta_stats.maximum() == max(alpha + beta)
def main():
args = list(map(float, sys.argv[1:]))
print('Statistics Functions')
print('Count:', len(args))
print('Mean:', mean(args))
print('Variance:', variance(args))
print('StdDev:', stddev(args))
print('Skewness:', skewness(args))
print('Kurtosis:', kurtosis(args))
fast_stats = FastStatistics()
for arg in args:
fast_stats.push(arg)
print()
print('FastStatistics')
print('Count:', len(fast_stats))
print('Mean:', fast_stats.mean())
print('Variance:', fast_stats.variance())
print('StdDev:', fast_stats.stddev())
print('Skewness:', fast_stats.skewness())
print('Kurtosis:', fast_stats.kurtosis())
core_stats = CoreStatistics()
for arg in args:
core_stats.push(arg)
print()
print('CoreStatistics')
print('Count:', len(core_stats))
print('Mean:', core_stats.mean())
print('Variance:', core_stats.variance())
print('StdDev:', core_stats.stddev())
print('Skewness:', core_stats.skewness())
print('Kurtosis:', core_stats.kurtosis())
fast_regr = FastRegression()
for index, arg in enumerate(args, 1):
fast_regr.push(index, arg)
print()
print('FastRegression')
print('Count:', len(fast_regr))
print('Slope:', fast_regr.slope())
print('Intercept:', fast_regr.intercept())
print('Correlation:', fast_regr.correlation())
core_regr = CoreRegression()
for index, arg in enumerate(args, 1):
core_regr.push(index, arg)
print()
print('CoreRegression')
print('Count:', len(core_regr))
print('Slope:', core_regr.slope())
print('Intercept:', core_regr.intercept())
print('Correlation:', core_regr.correlation())
