def main():
try:
args = get_args()
config = process_config(args.config)
tf_config = tf.ConfigProto()
tf_config.gpu_options.allow_growth = True
sess = tf.Session(config=tf_config)
ktf.set_session(sess)
create_dirs([config.callbacks.tensorboard_log_dir,
config.callbacks.checkpoint_dir,
config.path.chache_path])
print("Create the data generator.")
data_loader = SuperResolutionDataLoader(config)
print("Create the model.")
model = SuperResolutionModel(config)
print("Create the trainer.")
trainer = SuperResolutionTrainer(model.model,
data_loader.generate_train_data(),
config)
print("Start training...!")
trainer.train()
except Exception as err:
print("missing or invalid arguments: {0}".format(err))
exit(0)
def main():
# capture the config path from the run arguments
# then process the json configration file
try:
args = get_args()
config = process_config(args.config)
except Exception as e:
print(e)
print('Missing or invalid arguments')
exit(0)
# create the experiments dirs
create_dirs([config.summary_dir, config.checkpoint_dir])
# create tensorflow session
sess = tf.Session()
# create instance of the model you want
model = Model(config)
# load model if exist
model.load(sess)
# create your data generator
data = IVUSDataGenerator(config)
# create tensorboard logger
logger = Logger(sess, config)
# create trainer and path all previous components to it
trainer = SigmoidTrainer(sess, model, data, config, logger)
# train the model
trainer.train()
def train(cfg):
if cfg.COMM.OPTIM == "pso":
if cfg.COMM.MODE == "power":
optim = PSO_POWER(cfg)
elif cfg.COMM.MODE == "gauss_n4":
optim = PSO_GAUSS4(cfg)
elif cfg.COMM.MODE == "gauss_n6":
optim = PSO_GAUSS6(cfg)
else:
raise Exception(f"Invalid input function {cfg.COMM.MODE}.")
p = Pool(2)
results = p.map(optim.compute, range(cfg.COMM.EXEC))
for i, res in enumerate(results):
create_dirs(cfg.DATA.DIR + "param/")
np.savetxt(
cfg.DATA.DIR +
f"param/{i}_param_{cfg.COMM.OPTIM}_{cfg.COMM.MODE}_\
te{cfg.CALC.TE_str}_se{cfg.CALC.SE_str}.csv",
res,
delimiter=",",
)
logger.info("saved param at " + cfg.DATA.DIR +
f"param/{i}_param_{cfg.COMM.OPTIM}_{cfg.COMM.MODE}_\
te{cfg.CALC.TE_str}_se{cfg.CALC.SE_str}.csv")
elif cfg.COMM.OPTIM == "nsga2":
p = Pool(2)
optim = NSGA2(cfg)
results = p.map(optim.run, range(cfg.COMM.EXEC))
else:
raise Exception(f"Invalid optimizer {cfg.COMM.OPTIM}.")
def plot_graph(df, cfg):
create_dirs(cfg.DATA.DIR + "plot/")
output_file(cfg.DATA.DIR + f"plot/{0}_plot_pso_{cfg.COMM.MODE}_\
te{cfg.CALC.TE_str}_se{cfg.CALC.SE_str}.html")
width, height = 350, 250
fig1 = figure(width=width, plot_height=height, title="θ")
fig2 = figure(width=width, plot_height=height, title="dθ")
fig3 = figure(width=width, plot_height=height, title="ddθ")
fig4 = figure(width=width, plot_height=height, title="trq")
fig5 = figure(width=width, plot_height=height, title="w1")
fig6 = figure(width=width, plot_height=height, title="w2")
fig1.line(df["t"], df["θ"])
fig2.line(df["t"], df["dθ"])
fig3.line(df["t"], df["ddθ"])
fig4.line(df["t"], df["trq"])
fig5.line(df["t"], df["w1"])
fig6.line(df["t"], df["w2"])
fig = gridplot([
[fig1, fig4],
[fig2, fig5],
[fig3, fig6],
])
if cfg.COMM.PLOT:
show(fig)
def process_graphs(dt_start, dt_end):
str_dt = utils.get_str_dt(dt_start, dt_end)
out_dir = "{}/prints/{}/filtered/graph/".format(script_dir, str_dt)
utils.create_dirs([out_dir])
in_path = "{}/prints/{}/filtered/traceroute_per_mac.csv".format(
script_dir, str_dt)
servers = np.unique(pd.read_csv(in_path)["server"].values)
for traceroute_type in unsupervised_utils.iter_traceroute_types():
valid_traceroute_field, traceroute_field = \
cp_utils.get_traceroute_fields(traceroute_type)
for server in servers:
utils.create_dirs([
"{}/prints/{}/filtered/graph/".format(script_dir, str_dt),
"{}/prints/{}/filtered/graph/{}".format(
script_dir, str_dt, server)
])
out_dir = "{}/prints/{}/filtered/graph/{}".format(
script_dir, str_dt, server)
out_path = "{}/{}_graph.gv".format(out_dir, traceroute_field)
name_neigh = get_graph(dt_start, dt_end, valid_traceroute_field,
traceroute_field, server)
write_graph(out_path, name_neigh)
check_graph(out_dir, name_neigh, traceroute_field)
def main_train():
word_to_id, id_to_word, label_to_id, id_to_label = get_or_create_vocab(
FLAGS)
config = get_or_create_config(FLAGS, id_to_word)
# create the experiments dirs
create_dirs(
[config.tensorboard_dir, config.vocab_dir, config.checkpoint_dir])
# create tensorflow session
sess = tf.Session()
# create an instance of the model you want
model = DoubleLSTMModel(config)
# load model if exists
model.load(sess)
# create your data generator
train_data = DataGenerator(config, word_to_id, label_to_id,
config.train_file)
val_data = DataGenerator(config, word_to_id, label_to_id, config.val_file)
# create tensorboard logger
# logger = Logger(sess, config)
# create trainer and pass all the previous components to it
trainer = DoubleLSTMTrainer(sess, model, train_data, val_data, config,
id_to_label)
# here you train your model
trainer.train()
sess.close()
def run_test(cfg):
"""パラメータから直接テストを実行
"""
a = np.array(
str2list(
"6.455348882490680176e-06 2.205882734637162890e-02 1.144408684542324445e-01 -1.232142017960534019e+00 -2.722007589944505090e-01 -6.212503361141449298e-01"
))
print(f"param: {a}")
S = cycloid(a, cfg)
X1, X2 = RK4(S)
w1 = X1[0, :] * 2.7244
w2 = X2[0, :] * 2.7244
trq = torque(S, X1, X2)
df = pd.DataFrame({
"t": np.linspace(0, cfg.CALC.TEND, cfg.CALC.Nrk + 1),
"θ": S[0:2 * cfg.CALC.Nrk + 1:2, 0],
"dθ": S[0:2 * cfg.CALC.Nrk + 1:2, 1],
"ddθ": S[0:2 * cfg.CALC.Nrk + 1:2, 2],
"trq": trq,
"w1": w1,
"w2": w2,
})
create_dirs(cfg.DATA.DIR)
df.to_csv(cfg.DATA.DIR + f"{0}_output_{cfg.COMM.OPTIM}_{cfg.COMM.MODE}_\
te{cfg.CALC.TE}_se{cfg.CALC.SE}.csv")
# plot_graph(df, cfg)
print(f'ene: {energy(df["trq"], df["θ"])}\n')
def setup_logging(self):
self.experiments_root_dir = 'experiments'
self.config.model_name = const.get_model_name(self.config.model_name,
self.config)
self.config.summary_dir = os.path.join(
self.experiments_root_dir + "/" + self.config.log_dir + "/",
self.config.model_name)
self.config.checkpoint_dir = os.path.join(
self.experiments_root_dir + "/" + self.config.checkpoint_dir + "/",
self.config.model_name)
self.config.results_dir = os.path.join(
self.experiments_root_dir + "/" + self.config.result_dir + "/",
self.config.model_name)
#Flags
flags_list = [
'train', 'restore', 'plot', 'clustering', 'early_stopping', 'colab'
]
self.flags = utils.Config({
your_key: self.config.__dict__[your_key]
for your_key in flags_list
})
# create the experiments dirs
utils.create_dirs([
self.config.summary_dir, self.config.checkpoint_dir,
self.config.results_dir
])
utils.save_args(self.config.__dict__, self.config.summary_dir)
def __init__(self, config, run_id=None, verbosity=0):
"""
Class to parse configuration json file. Handles hyper-parameters for training, initializations of modules,
checkpoint saving and logging module.
:param config: Dict containing configurations.
:param run_id: Unique Identifier for training processes. Timestamp is being used as default
:param verbosity: default 0.
"""
# 1\ Define the config and run_id.
self._config = config
self._run_id = str(run_id)
if run_id is 'None':
self._run_id = datetime.now().strftime(r'%m%d_%H%M%S')
self._verbosity = verbosity
# 2\ Set _save_dir, _checkpoint_dir and _log_dir where checkpoints and logs will be saved.
save_dir = './result/'
self._result_dir = os.path.join(save_dir, self.config['name'] + '/',
self._run_id + '/')
self._checkpoint_dir = os.path.join(save_dir,
self.config['name'] + '/',
self._run_id + '/', 'checkpoints/')
self._summary_dir = os.path.join(save_dir, self.config['name'] + '/',
self._run_id + '/', 'logs/')
# 3\ Create directory for saving checkpoints and log.
create_dirs([self.result_dir, self._checkpoint_dir, self.summary_dir])
# 4\ Save relative config file to the relative dir
write_json(self.config, os.path.join(self.result_dir, 'config.json'))
self.config['trainer']['args']['verbosity'] = verbosity
def main():
# get json configuration filepath from the run argument
# process the json configuration file
try:
args = get_args()
config, log_dir, checkpoint_dir = process_config(args.config)
except:
print('missing or invalid arguments')
print('Unexpected error:', sys.exc_info()[0])
# create the experiment directories
create_dirs([log_dir, checkpoint_dir])
print('Create the data generator')
data_loader = WikiArtDataLoader(config)
print('Create the model')
model = ResNet50AttrModel(config)
print('model ready loading data now')
print('Create the trainer')
trainer = ResNet50ModelTrainer(model.model, data_loader.get_train_data(), data_loader.get_val_data(), config, log_dir, checkpoint_dir)
print('Start training the model.')
trainer.train()
def write_csvs(dt_dir, dt_start, dt_end, cursor, collection):
for cnt, doc in enumerate(cursor):
print "{}, {}".format(cnt, utils.get_str_dt(dt_start, dt_end))
if valid_doc(doc):
mac = doc["_id"]["mac"]
server = doc["host"]
utils.create_dirs([
"{}/{}".format(script_dir, dt_dir),
"{}/{}/{}/".format(script_dir, dt_dir, server)
])
out_path = "{}/{}/{}/{}.csv".format(script_dir, dt_dir, server,
mac)
if not os.path.exists(out_path):
with open(out_path, "w") as f:
l = ("dt,uf,server_ip,loss,latency,throughput_up,"
"throughput_down,nominal_up,nominal_down,"
"loss_cross_traffic_up,loss_cross_traffic_down,"
"latency_cross_traffic_up,latency_cross_traffic_down,"
"throughput_up_cross_traffic_up,"
"throughput_up_cross_traffic_down,"
"throughput_down_cross_traffic_up,"
"throughput_down_cross_traffic_down,"
"traceroute\n")
f.write(l)
uf = get_uf(doc)
server = doc["host"]
dt = dt_procedures.from_utc_to_sp(doc["_id"]["date"])
server_ip = get_server_ip(doc)
(loss, loss_cross_traffic_up, loss_cross_traffic_down) = \
get_loss(doc)
(latency, latency_cross_traffic_up,
latency_cross_traffic_down) = get_latency(doc)
(throughput_up, nominal_up, throughput_up_cross_traffic_up,
throughput_up_cross_traffic_down) = get_throughput_up(doc)
(throughput_down, nominal_down, throughput_down_cross_traffic_up,
throughput_down_cross_traffic_down) = get_throughput_down(doc)
traceroute = get_traceroute(doc)
l = "{}" + ",{}" * 16 + ",\"{}\"\n"
l = l.format(dt, uf, server_ip, loss, latency, throughput_up,
throughput_down, nominal_up, nominal_down,
loss_cross_traffic_up, loss_cross_traffic_down,
latency_cross_traffic_up, latency_cross_traffic_down,
throughput_up_cross_traffic_up,
throughput_up_cross_traffic_down,
throughput_down_cross_traffic_up,
throughput_down_cross_traffic_down, traceroute)
with open(out_path, "a") as f:
f.write(l)
def process_config():
args = get_args()
config = get_edict()
config.update(vars(args))
create_dirs([
args.CTPN_MODEL, args.CTPN_LOGGER, args.DENSENET_LOGGER,
args.SAVED_PATH
])
return edict(config)
def main():
# Capture the config path from the run arguments then process the json configuration file
try:
args = utils.get_args()
config = process_config(args.config)
except:
print("missing or invalid arguments")
exit(0)
if not os.path.exists(config.dataset_path_train):
print(config.dataset_path_train)
print('ERROR: Dataset not found')
exit(0)
## Set log levels
tf.logging.set_verbosity(tf.logging.INFO)
## Set seed values to reproduce results
# random.seed(7)
# np.random.seed(7)
## Create output dirs
# utils.remove_dirs([os.path.join('output', config.exp_name)])
utils.create_dirs([
config.summary_dir, config.checkpoint_dir, config.tfrecords_path_train,
config.tfrecords_path_test
])
## GPU
sess_config = tf.ConfigProto()
sess_config.gpu_options.allow_growth = True
## Create tensorflow session
with tf.Session(config=sess_config) as sess:
## Debug
# sess = tf_debug.LocalCLIDebugWrapperSession(sess)
## Create tensorboard logger
logger = TFLogger(sess, config)
## Create TF Records
TFRecordsKnifeySpoony(config)
## Create data generator using TF Records
data = DataGeneratorKnifeySpoony(config)
## Create model
model = ModelKnifeySpoony(config)
## Create Trainer
trainer = TrainerKnifeySpoony(sess, model, data, config, logger)
## Train model
trainer.train()
trainer.evaluate()
trainer.predict()
def __init__(self, config):
self.config = config
## Setup
utils.remove_dirs(
[config.tfrecords_path_train, config.tfrecords_path_test])
utils.create_dirs(
[config.tfrecords_path_train, config.tfrecords_path_test])
def run(self):
self.train()
utils.create_dirs([
"{}/plots/".format(script_dir),
"{}/plots/{}".format(script_dir, self.__class__.__name__)
])
self.write_model("{}/plots/{}/model.txt".format(
script_dir, self.__class__.__name__))
self.test()
self.sample(500)
def __init__(self, **kwrds):
self.config = utils.Config(copy.deepcopy(const.config))
for key in kwrds.keys():
assert key in self.config.keys(), '{} is not a keyword, \n acceptable keywords: {}'.\
format(key, self.config.keys())
self.config[key] = kwrds[key]
self.experiments_root_dir = 'experiments'
utils.create_dirs([self.experiments_root_dir])
self.config.model_name = const.get_model_name(self.config.model_type,
self.config)
self.config.checkpoint_dir = os.path.join(
self.experiments_root_dir + "/" + self.config.checkpoint_dir + "/",
self.config.model_name)
self.config.summary_dir = os.path.join(
self.experiments_root_dir + "/" + self.config.summary_dir + "/",
self.config.model_name)
self.config.log_dir = os.path.join(
self.experiments_root_dir + "/" + self.config.log_dir + "/",
self.config.model_name)
utils.create_dirs([
self.config.checkpoint_dir, self.config.summary_dir,
self.config.log_dir
])
load_config = {}
try:
load_config = utils.load_args(self.config.model_name,
self.config.summary_dir)
self.config.update(load_config)
self.config.update({
key: const.config[key]
for key in ['kinit', 'bias_init', 'act_out', 'transfer_fct']
})
print('Loading previous configuration ...')
except:
print('Unable to load previous configuration ...')
utils.save_args(self.config.dict(), self.config.model_name,
self.config.summary_dir)
if self.config.plot:
self.latent_space_files = list()
self.latent_space3d_files = list()
self.recons_files = list()
if hasattr(self.config, 'height'):
try:
self.config.restore = True
self.build_model(self.config.height, self.config.width,
self.config.num_channels)
except:
self.isBuild = False
else:
self.isBuild = False
def plot_latencies_traceroute(dt_start, dt_end, preprocess_args):
str_dt = utils.get_str_dt(dt_start, dt_end)
in_path = "{}/prints/{}/filtered/traceroute_per_mac.csv".format(script_dir,
str_dt)
df = pd.read_csv(in_path)
for _, row, in df.iterrows():
if row["valid_cnt_samples"]:
in_path = utils.get_in_path(row["server"], row["mac"], dt_start,
dt_end)
ts_traceroute = TimeSeries(in_path=in_path, metric="traceroute",
dt_start=dt_start, dt_end=dt_end)
for traceroute_type in unsupervised_utils.iter_traceroute_types():
valid_traceroute_field, traceroute_field = \
cp_utils.get_traceroute_fields(traceroute_type)
if row[valid_traceroute_field]:
traceroute = ast.literal_eval(row[traceroute_field])
name_ts = get_ts_per_name(traceroute_type, ts_traceroute,
dt_start, dt_end)
dir_path = ("{}/plots/paths/{}/{}/{}/{}".
format(script_dir, str_dt, "latency",
traceroute_type, row["server"]))
traceroute_path = "/".join(map(str,
list(reversed(traceroute))))
dir_path = "{}/{}".format(dir_path, traceroute_path)
utils.create_dirs(["{}/traceroute_latencies/".
format(dir_path),
"{}/traceroute_latencies/{}".
format(dir_path, row["mac"])])
for i in range(len(traceroute) - 1):
name = traceroute[i][0][0]
traceroute_path = "hop{}_{}".format(str(i).zfill(2),
name)
out_path = ("{}/traceroute_latencies/{}/{}.png".
format(dir_path, row["mac"],
traceroute_path))
ts_preprocessed = name_ts[name].copy()
cp_utils.preprocess(ts_preprocessed, preprocess_args)
# plot_procedures.plot_ts_share_x(
# name_ts[name],
# ts_preprocessed,
# out_path,
# plot_type2="scatter",
# title1="raw",
# title2="median filtered",
# default_ylabel=True)
ts_preprocessed.metric = "latency"
plot_procedures.plot_ts(ts_preprocessed, out_path,
title="median filtered")
def print_cps(dt_start, dt_end, dir_model, metric, preprocess_args):
str_dt = utils.get_str_dt(dt_start, dt_end)
utils.create_dirs(["{}/prints/".format(script_dir),
"{}/prints/{}".format(script_dir, str_dt),
"{}/prints/{}/filtered".format(script_dir, str_dt),
"{}/prints/{}/filtered/{}".format(script_dir, str_dt,
metric)])
out_path = "{}/prints/{}/filtered/{}/cps_per_mac.csv".format(script_dir,
str_dt,
metric)
with open(out_path, "w") as f:
f.write("server,mac,cp_dts,type_cps,seg_means\n")
in_path_dir = ("{}/change_point/models/{}/plots/unsupervised/{}/{}".
format(base_dir, dir_model, str_dt, metric))
cnt = 0
for file_name in os.listdir(in_path_dir):
if ".csv" in file_name:
cnt += 1
print "cnt={}".format(cnt)
server = file_name.split("server")[1].split("_")[0]
mac = file_name.split("mac")[1].split("_")[0]
dt_cps = []
id_cps = []
df = pd.read_csv("{}/{}".format(in_path_dir, file_name))
for idx, row in df.iterrows():
dt_cps.append(row["dt"])
id_cps.append(row["dt_id"])
in_path = utils.get_in_path(server, mac, dt_start, dt_end)
ts = TimeSeries(in_path, metric, dt_start, dt_end)
cp_utils.preprocess(ts, preprocess_args)
seg_means = []
type_cps = []
if id_cps:
mean1 = np.mean(ts.y[0:id_cps[0]])
seg_means.append(mean1)
for i in range(1, len(id_cps)):
mean2 = np.mean(ts.y[id_cps[i - 1]:id_cps[i]])
seg_means.append(mean2)
update_type_cps(type_cps, mean1, mean2, metric)
mean1 = mean2
mean2 = np.mean(ts.y[id_cps[-1]:-1])
seg_means.append(mean2)
update_type_cps(type_cps, mean1, mean2, metric)
f.write("{},{},\"{}\",\"{}\",\"{}\"\n".format(server, mac,
dt_cps, type_cps,
seg_means))
def run(dataset, cmp_class_args, preprocess_args, param, metric):
model = SlidingWindowsOnline(preprocess_args=preprocess_args,
metric=metric,
**param)
utils.create_dirs([
"{}/online/".format(script_dir), "{}/online/plots/".format(script_dir),
"{}/online/plots/{}".format(script_dir, dataset),
"{}/online/plots/{}/{}".format(script_dir, dataset, metric)
])
out_dir_path = "{}/online/plots/{}/{}".format(script_dir, dataset, metric)
model.plot_all(dataset, out_dir_path, cmp_class_args)
def run(dataset, cmp_class_args, preprocess_args, param, metric):
model = SegmentNeighbourhood(preprocess_args=preprocess_args,
metric=metric,
**param)
utils.create_dirs([
"{}/plots/".format(script_dir),
"{}/plots/{}/".format(script_dir, dataset),
"{}/plots/{}/{}".format(script_dir, dataset, metric)
])
out_dir_path = "{}/plots/{}/{}".format(script_dir, dataset, metric)
model.plot_all(dataset, out_dir_path, cmp_class_args)
def run(dataset, cmp_class_args, preprocess_args, param, metric):
model = GaussianHMM(preprocess_args=preprocess_args,
metric=metric,
**param)
utils.create_dirs([
"{}/gaussian/".format(script_dir),
"{}/gaussian/plots/".format(script_dir),
"{}/gaussian/plots/{}".format(script_dir, dataset),
"{}/gaussian/plots/{}/{}".format(script_dir, dataset, metric)
])
out_dir_path = "{}/gaussian/plots/{}/{}".format(script_dir, dataset,
metric)
model.plot_all(dataset, out_dir_path, cmp_class_args)
def print_valid_nodes():
df = pd.read_csv("{}/input/probes_info.csv".format(base_dir), sep=" ")
l = []
for node in df["NODE"].unique():
cnt_node = len(df[df["NODE"] == node])
if is_valid_node(node, cnt_node):
l.append((cnt_node, node))
l.sort(reverse=True)
utils.create_dirs(["{}/prints".format(script_dir)])
with open("{}/prints/valid_nodes.csv".format(script_dir), "w") as f:
f.write("node,cnt\n")
for tp in l:
f.write("{},{}\n".format(tp[1], tp[0]))
def print_all(dt_start, dt_end, mac_node):
str_dt = utils.get_str_dt(dt_start, dt_end)
utils.create_dirs([
"{}/prints".format(script_dir),
"{}/prints/{}".format(script_dir, str_dt),
"{}/prints/{}/filtered".format(script_dir, str_dt),
"{}/prints/{}/not_filtered".format(script_dir, str_dt)
])
# print_macs_per_name(dt_start, dt_end, mac_node)
# print_names_per_mac(dt_start, dt_end, mac_node)
# print_name_ips(dt_start, dt_end)
print_traceroute_per_mac(dt_start, dt_end)
print_traceroute_per_mac_filtered(dt_start, dt_end)
def main():
# get json configuration filepath from the run argument
# process the json configuration file
args = get_args()
config = process_config(args.config)
# create the experiment directories
log_dir, checkpoint_dir = create_dirs(config)
print('Create the data generator')
data_loader = DataLoader(config)
print('Create the model')
model = CycleGANAttrModel(config, config['weights_path'])
model.build_model()
print('model ready loading data now')
print('Create the trainer')
trainer = CycleGANModelTrainer(model, data_loader.get_trainA_data(),
data_loader.get_trainB_data(),
data_loader.get_testA_data(),
data_loader.get_testB_data(), config,
log_dir, checkpoint_dir)
# print('Start training the model.')
trainer.train()
def plot(dt_start, dt_end, metric):
dt_dir = utils.get_dt_dir(dt_start, dt_end)
str_dt = utils.get_str_dt(dt_start, dt_end)
utils.create_dirs([
"{}/{}".format(script_dir, str_dt),
"{}/{}/{}".format(script_dir, str_dt, metric)
])
for server, mac, in_path in utils.iter_server_mac(dt_dir, True):
out_file_name = utils.get_out_file_name(server, mac, dt_start, dt_end)
out_path = "{}/{}/{}/{}.png".format(script_dir, str_dt, metric,
out_file_name)
# comparison between not filtered and filtered
ts = TimeSeries(in_path, metric, dt_start, dt_end)
ts_filter = TimeSeries(in_path, metric, dt_start, dt_end)
ts_filter.percentile_filter(win_len=5, p=0.5)
# if len(ts_filter.y) > 100:
# plot_procedures.plot_stl_decomposition(ts_filter,
# "median_filtered",
# out_path)
# comparison between with cross traffic and without
# ts = TimeSeries(in_path, metric, dt_start, dt_end)
# ts.percentile_filter(win_len=13, p=0.5)
# ts_filter = TimeSeries(in_path, metric, dt_start, dt_end,
# cross_traffic_thresh=0)
# ts_filter.percentile_filter(win_len=13, p=0.5)
# plot_procedures.plot_ts_share_x(ts, ts_filter, out_path,
# compress=True,
# plot_type2="scatter",
# title1="raw",
# title2="median filtered",
# default_ylabel=True,
# xlabel="$i$")
ylabel = plot_procedures.get_default_ylabel(ts)
plot_procedures.plot_ts(ts_filter,
out_path,
ylabel=ylabel,
compress=False,
title="median filtered")
def plot_best_hyperparameters():
client = pymongo.MongoClient()
collection = client["change_point"]["random_search"]
model_classes = [
SegmentNeighbourhood, SlidingWindowsOffline, SlidingWindowsOnline,
BayesianOffline, BayesianOnline, GaussianHMM, DiscreteHMM
]
metric = cmp_class.f_1_score
datasets = ["*****@*****.**"]
metric = metric.__name__
for dataset in datasets:
for model_class in model_classes:
cursor = collection.find({
"$and": [{
"dataset": dataset
}, {
"model_class": model_class.__name__
}]
})
cursor = cursor.sort("metrics.{}".format(metric),
pymongo.DESCENDING).limit(1)
for doc in cursor:
cmp_class_args = doc["cmp_class_args"]
preprocess_args = doc["preprocess_args"]
params = get_params(doc["model_class"], doc["params"])
utils.create_dirs([
"{}/plots/".format(script_dir),
"{}/plots/{}/".format(script_dir, dataset),
"{}/plots/{}/{}/".format(script_dir, dataset,
model_class.__name__)
])
out_dir_path = "{}/plots/{}/{}/".format(
script_dir, dataset, model_class.__name__)
with open("{}/params.json".format(out_dir_path), "w") as f:
f.write(json_util.dumps(doc, indent=4, sort_keys=True))
model = model_class(preprocess_args=preprocess_args, **params)
model.plot_all(dataset, out_dir_path, cmp_class_args)
def plot_per_node(dt_start, dt_end, metric, only_unique_traceroute):
dt_dir = utils.get_dt_dir(dt_start, dt_end)
str_dt = utils.get_str_dt(dt_start, dt_end)
utils.create_dirs([
"{}/plots/".format(script_dir), "{}/plots/nodes".format(script_dir),
"{}/plots/nodes/{}".format(script_dir, str_dt),
"{}/plots/nodes/{}/{}".format(script_dir, str_dt, metric)
])
valid_nodes = read_input.get_valid_nodes()
mac_node = read_input.get_mac_node()
macs_unique_traceroute = read_input.get_macs_traceroute_filter(
dt_start, dt_end, "filtered")
for server, mac, in_path in utils.iter_server_mac(dt_dir, True):
if only_unique_traceroute and (mac not in macs_unique_traceroute):
continue
if mac_node[mac] in valid_nodes:
utils.create_dirs([
"{}/plots/nodes/{}/{}/{}".format(script_dir, str_dt, metric,
mac_node[mac])
])
out_file_name = utils.get_out_file_name(server, mac, dt_start,
dt_end)
out_path = ("{}/plots/nodes/{}/{}/{}/{}.png".format(
script_dir, str_dt, metric, mac_node[mac], out_file_name))
ts = TimeSeries(in_path, metric, dt_start, dt_end)
ts_filter = TimeSeries(in_path, metric, dt_start, dt_end)
ts_filter.percentile_filter(win_len=13, p=0.5)
plot_procedures.plot_ts_share_x(ts,
ts_filter,
out_path,
compress=False,
plot_type2="scatter")
def plot_dataset():
utils.create_dirs(["{}/plots/".format(script_dir)])
in_dir = "{}/change_point/input/".format(base_dir)
ts_cnt = get_cnt_class_per_ts(in_dir)
for dataset in os.listdir(in_dir):
if os.path.isdir("{}/{}".format(in_dir, dataset)) and \
(dataset != "unsupervised"):
df = pd.read_csv("{}/{}/dataset.csv".format(in_dir, dataset))
for idx, row in df.iterrows():
print "dataset={}, idx={}".format(dataset, idx)
ts = cp_utils.get_ts(row, {"filter_type": "none"}, "loss")
if not ts.y:
continue
correct = cp_utils.from_str_to_int_list(
row["change_points_ids"])
_, dt_start, dt_end = cp_utils.unpack_pandas_row(row)
ts_key = (row["server"], row["mac"], row["dt_start"],
row["dt_end"])
out_file_name = utils.get_out_file_name(
row["server"], row["mac"], dt_start, dt_end)
out_dir = "{}/plots/cnt{}_{}".format(script_dir,
ts_cnt[ts_key],
out_file_name)
out_path = "{}/{}.png".format(out_dir, dataset)
utils.create_dirs([out_dir])
plot_procedures.plot_ts(ts,
out_path,
ylim=[-0.02, 1.02],
dt_axvline=np.asarray(ts.x)[correct],
compress=True,
ylabel="loss fraction",
xlabel="$i$",
title="raw")
def process_config(yaml_file):
"""
.
"""
config = get_config_from_yaml(yaml_file)
print("THE CONFIGURATION of this experiment")
pprint(config)
try:
print(70 * "-")
print(f"The name of this experiment is: {config.exp_name}")
print(70 * "-")
except AttributeError:
print("ERROR!! Please provide the exp_name in yaml config file...")
exit(-1)
# Setup some important directories to be used for that run
dirs_to_create = []
if config.t_c.want_log:
config.t_c.summary_dir = os.path.join("experiments", config.exp_name,
"summaries")
dirs_to_create += [config.t_c.summary_dir]
config.t_c.checkpoint_dir = os.path.join("experiments", config.exp_name,
"checkpoints")
config.t_c.out_dir = os.path.join("experiments", config.exp_name, "out")
config.t_c.log_dir = os.path.join("experiments", config.exp_name, "logs")
dirs_to_create += [
config.t_c.checkpoint_dir, config.t_c.out_dir, config.t_c.log_dir
]
create_dirs(dirs_to_create)
return config
def create_dataset_unsupervised(dt_start, dt_end):
"""
all [dt_start, dt_end) must be in the same month.
datetimes must represent days
"""
str_dt = utils.get_str_dt(dt_start, dt_end)
utils.create_dirs([
"{}/change_point/input/unsupervised/".format(base_dir),
"{}/change_point/input/unsupervised/{}".format(base_dir, str_dt)
])
out_path = "{}/unsupervised/{}/dataset.csv".format(script_dir, str_dt)
with open(out_path, "w") as f:
f.write("email,mac,server,dt_start,dt_end,change_points,"
"change_points_ids\n")
in_path = ("{}/change_point/unsupervised/prints/{}/filtered/"
"traceroute_per_mac.csv".format(base_dir, str_dt))
df = pd.read_csv(in_path)
for idx, row in df.iterrows():
if include_in_dataset(row):
f.write("{},{},{},{},{},\"\",\"\"\n".format(
str_dt, row["mac"], row["server"], dt_start, dt_end))
