def debug_updated_weights(opts, steps, weights, data):
""" Various debug plots for updated weights of training points.
"""
assert data.num_points == len(weights), 'Length mismatch'
ws_and_ids = sorted(zip(weights, range(len(weights))))
num_plot = 20 * 16
if num_plot > len(weights):
return
ids = [_id for w, _id in ws_and_ids[:num_plot]]
plot_points = data.data[ids]
metrics = metrics_lib.Metrics()
metrics.make_plots(opts, steps, None, plot_points, prefix='d_least_')
ids = [_id for w, _id in ws_and_ids[-num_plot:]]
plot_points = data.data[ids]
metrics = metrics_lib.Metrics()
metrics.make_plots(opts, steps, None, plot_points, prefix='d_most_')
plt.clf()
ax1 = plt.subplot(211)
ax1.set_title('Weights over data points')
plt.plot(range(len(weights)), sorted(weights))
plt.axis([0, len(weights), 0., 2. * np.max(weights)])
if data.labels is not None:
all_labels = np.unique(data.labels)
w_per_label = -1. * np.ones(len(all_labels))
for _id, y in enumerate(all_labels):
w_per_label[_id] = np.sum(weights[np.where(data.labels == y)[0]])
ax2 = plt.subplot(212)
ax2.set_title('Weights over labels')
plt.scatter(range(len(all_labels)), w_per_label, s=30)
filename = 'data_w{:02d}.png'.format(steps)
create_dir(opts['work_dir'])
plt.savefig(o_gfile((opts["work_dir"], filename), 'wb'))
def __init__(self, metric_names):
self.model_name = P.MODEL_ID
self.setup_folders()
initialize_logger(
os.path.join(self.model_folder, 'log.txt').format(self.model_name))
P.write_to_file(os.path.join(self.model_folder, 'config.ini'))
logging.info(P.to_string())
self.train_metrics = metrics.Metrics('train', metric_names,
P.N_CLASSES)
self.val_metrics = metrics.Metrics('validation', metric_names,
P.N_CLASSES)
self.epoch = -1
def run(args):
appMetrics = metrics.Metrics(args)
data = data_set.DcsData(args, appMetrics)
data.run()
appMetrics.write()
def Classify(expdir):
saver.restore(session, os.path.join(expdir, 'model.bin'))
context_var = params.context_vars[-1]
placeholder = model.context_placeholders[context_var]
lang_vocab = context_vocabs[context_var]
vocab_subset = [w for w in lang_vocab.GetWords() if w != '<UNK>']
print vocab_subset
print 'preparing dataset'
dataset.Filter(vocab_subset, context_var)
dataset.Prepare(vocab, context_vocabs)
results = []
all_labels = []
all_preds = []
for pos in xrange(dataset.GetNumBatches()):
if pos % 10 == 0:
print pos
batch = dataset.GetNextBatch()
feed_dict = GetFeedDict(batch, use_dropout=False)
labels = np.array(feed_dict[placeholder])
def GetCosts():
costs = []
if use_nce_loss:
feed_dict[placeholder][:] = lang_vocab[vocab_subset[0]]
result = session.run([model.per_sentence_loss] + model.sampled_values, feed_dict)
sentence_costs, sampled_vals = result[0], result[1:]
costs.append(sentence_costs)
# reuse the sampled values
for i in range(len(model.sampled_values)):
feed_dict[model.sampled_values[i][0]] = sampled_vals[i][0]
feed_dict[model.sampled_values[i][1]] = sampled_vals[i][1]
feed_dict[model.sampled_values[i][2]] = sampled_vals[i][2]
for i in range(1, len(vocab_subset)):
feed_dict[placeholder][:] = lang_vocab[vocab_subset[i]]
costs.append(session.run(model.per_sentence_loss, feed_dict))
else: # full softmax
for i in range(len(vocab_subset)):
feed_dict[placeholder][:] = lang_vocab[vocab_subset[i]]
costs.append(session.run(model.per_sentence_loss, feed_dict))
return np.array(costs)
costs = GetCosts()
for label, c_array in zip(labels, costs.T):
d = dict(zip(vocab_subset, c_array))
d['label'] = lang_vocab[label]
results.append(d)
predictions = np.argmin(costs, 0)
all_preds += [lang_vocab[vocab_subset[x]] for x in predictions]
all_labels += list(labels)
df = pandas.DataFrame(results)
df.to_csv(os.path.join(expdir, 'classify.csv'))
metrics.Metrics([lang_vocab[i] for i in all_preds],
[lang_vocab[i] for i in all_labels])
def UnigramClassify(expdir):
# turn the model into a linear classifier by using the softmax bias
saver.restore(session, os.path.join(expdir, 'model.bin'))
probs = tf.nn.softmax(model.base_bias +
tf.transpose(model.bias_tables['subreddit']))
log_probs = tf.log(probs).eval(session=session)
lang_vocab = context_vocabs['subreddit']
vocab_subset = lang_vocab.GetWords()
print 'preparing dataset'
dataset.Prepare(vocab, context_vocabs)
preds = []
labels = []
for pos in xrange(dataset.GetNumBatches()):
if pos % 10 == 0:
print pos
batch = dataset.GetNextBatch()
for i in xrange(len(batch)):
row = batch.iloc[i]
scores = np.zeros(len(vocab_subset))
for word_id in row.text[1:row.seq_lens]:
scores += log_probs[:, word_id]
preds.append(np.argmax(scores))
labels.append(row['subreddit'])
metrics.Metrics([lang_vocab[i] for i in preds],
[lang_vocab[i] for i in labels])
def output(self): if not self.localize_output: localization.disable() terminal.skip_escapes(not sys.stdout.isatty()) terminal.set_stdout_encoding() previous_directory = os.getcwd() os.chdir(self.repo) absolute_path = basedir.get_basedir_git() os.chdir(absolute_path) format.output_header() outputable.output(changes.ChangesOutput(self.hard)) if changes.get(self.hard).get_commits(): outputable.output(blame.BlameOutput(self.hard, self.useweeks)) if self.timeline: outputable.output(timeline.Timeline(changes.get(self.hard), self.useweeks)) if self.include_metrics: outputable.output(metrics.Metrics()) if self.responsibilities: outputable.output(responsibilities.ResponsibilitiesOutput(self.hard, self.useweeks)) outputable.output(filtering.Filtering()) if self.list_file_types: outputable.output(extensions.Extensions()) format.output_footer() os.chdir(previous_directory)
def main(argv):
"""Main body of the program."""
parser = argparse.ArgumentParser(prog=argv[0])
parser.add_argument('--port',
help='HTTP server port',
type=int,
default=8091)
parser.add_argument('--config',
help='Configuration file (INI file)',
default='config.ini')
parser.add_argument('--log_level',
help='Logging level (DEBUG, INFO, WARNING, ERROR)',
type=str,
default='INFO')
parser.add_argument(
'--include_inactive_devices',
help=
'Do not use; this flag has no effect and remains for compatibility only',
action='store_true')
args = parser.parse_args()
try:
level = getattr(logging, args.log_level)
except AttributeError:
print(f'Invalid --log_level: {args.log_level}')
sys.exit(-1)
args = parser.parse_args()
logging.basicConfig(
format=
'%(asctime)s [%(name)24s %(thread)d] %(levelname)10s %(message)s',
datefmt='%Y/%m/%d %H:%M:%S',
level=level)
logger.info('Starting up on port=%s', args.port)
if args.include_inactive_devices:
logger.warning(
'--include_inactive_devices is now inoperative and will be removed in a future release'
)
try:
cfg = config.Config(args.config)
except:
logger.exception('Could not load configuration: %s', args.config)
sys.exit(-1)
devices = cfg.devices
if len(devices) == 0:
logger.fatal(
'No devices configured; please re-run this program with --create_device_cache.'
)
sys.exit(-2)
prometheus_client.start_http_server(args.port)
connect.ConnectionManager(metrics.Metrics().update, devices, cfg.hosts)
_sleep_forever()
def __init__(self):
# 环境初始化
self.global_arg = arg.init_global_arg()
env_arg = arg.init_env_arg(self.global_arg)
# 增加nk的一个读入操作
self.main_env = Env(env_arg)
for model_type in ['st', 'ed']:
if all_config['checkpoint']['env'][model_type]['enable']:
self.main_env.nkmodel_load(all_config['checkpoint']['env']['path'], model_type)
self.main_env.nkmodel_save(all_config["nkmodel_path"][model_type], model_type)
# 个体初始化
self.agents = []
csv_head_agent = ['agent_no'] + ['st_state'] + ['st_value'] + ['insight'] + ['xplr'] + ['xplt'] + ['enable']
moniter.AppendToCsv(csv_head_agent, all_config['agent_csv_path'])
for i in range(self.global_arg["Nagent"]):
# 个体随机初始位置
start_st_label = [randint(0, self.main_env.P - 1) for j in range(self.main_env.N)]
state_start = State(start_st_label)
self.agents.append(Agent(arg.init_agent_arg(self.global_arg,
self.main_env.arg),
self.main_env))
self.agents[i].state_now = deepcopy(state_start)
self.agents[i].agent_id = i
# 去除了一开始给一个全局area，改为添加一个包含起点的点area
start_area = Area(self.agents[i].state_now, [False] * self.main_env.N, 0)
start_area.info = get_area_sample_distr(env=self.main_env, area=start_area, state=self.agents[i].state_now,
T_stmp=0, sample_num=1, dfs_r=1)
start_area.sign = Sign(i, 0, 'start')
self.agents[i].renew_m_info(start_area, 0)
self.a_plan = None
logging.info("state:%s, st_value:%s,insight:%.5s ,xplr:%.5s, xplt:%.5s, enable:%.5s" % (
str(self.agents[i].state_now),
self.main_env.getValue(self.agents[i].state_now, 0),
self.agents[i].agent_arg['a']['insight'],
self.agents[i].agent_arg['a']['xplr'],
self.agents[i].agent_arg['a']['xplt'],
self.agents[i].agent_arg['a']['enable']))
# 记录agent信息
csv_info_agent = ['agent%d' % i] \
+ [self.agents[i].state_now] \
+ [self.main_env.getValue(self.agents[i].state_now, 0)] \
+ [self.agents[i].agent_arg['a']['insight']] \
+ [self.agents[i].agent_arg['a']['xplr']] \
+ [self.agents[i].agent_arg['a']['xplt']] \
+ [self.agents[i].agent_arg['a']['enable']]
moniter.AppendToCsv(csv_info_agent, all_config['agent_csv_path'])
# 社会网络初始化
soclnet_arg = arg.init_soclnet_arg(self.global_arg, env_arg)
self.socl_net = SoclNet(soclnet_arg)
self.socl_net.new_flat_init() # 修改初始化方法
# self.socl_net.flat_init()
if all_config['checkpoint']['socl_network']['enable']:
self.socl_net.power_load(all_config['checkpoint']['socl_network']['power'])
self.socl_net.relat_load(all_config['checkpoint']['socl_network']['relat'])
self.record = Record()
self.metric = metrics.register_all_metrics(metrics.Metrics())
def output(self):
if not self.localize_output:
localization.disable()
terminal.skip_escapes(not sys.stdout.isatty())
terminal.set_stdout_encoding()
previous_directory = os.getcwd()
os.chdir(self.repo)
isbare = subprocess.Popen("git rev-parse --is-bare-repository",
shell=True,
bufsize=1,
stdout=subprocess.PIPE).stdout
isbare = isbare.readlines()
self.isbare = (isbare[0].decode("utf-8", "replace").strip() == "true")
absolute_path = ""
if self.isbare:
absolute_path = subprocess.Popen("git rev-parse --git-dir",
shell=True,
bufsize=1,
stdout=subprocess.PIPE).stdout
else:
absolute_path = subprocess.Popen("git rev-parse --show-toplevel",
shell=True,
bufsize=1,
stdout=subprocess.PIPE).stdout
absolute_path = absolute_path.readlines()
if len(absolute_path) == 0:
sys.exit(_("Unable to determine absolute path of git repository."))
os.chdir(absolute_path[0].decode("utf-8", "replace").strip())
format.output_header()
outputable.output(changes.ChangesOutput(self.hard))
if changes.get(self.hard).get_commits():
outputable.output(blame.BlameOutput(self.hard))
if self.timeline:
outputable.output(
timeline.Timeline(changes.get(self.hard), self.useweeks))
if self.include_metrics:
outputable.output(metrics.Metrics())
if self.responsibilities:
outputable.output(
responsibilities.ResponsibilitiesOutput(self.hard))
outputable.output(filtering.Filtering())
if self.list_file_types:
outputable.output(extensions.Extensions())
format.output_footer()
os.chdir(previous_directory)
def collect_metrics(self):
"""Sample system metrics and populate a metrics object suitable for publishing to Device Defender."""
metrics_current = metrics.Metrics(last_metric=self._last_metric)
self.network_stats(metrics_current)
self.listening_ports(metrics_current)
self.network_connections(metrics_current)
self._last_metric = metrics_current
return metrics_current
def init_matchers(nlp_model, database_df, destination_urls):
d = dict()
mtr = m.Metrics('matcher')
for url in destination_urls:
mtr.start()
phrase_list = list(database_df['Lemma'][database_df['URL'] == url])
phrase_patterns = [nlp_model(text) for text in phrase_list]
matcher = PhraseMatcher(nlp_model.vocab, attr="LEMMA")
matcher.add(url, None, *phrase_patterns)
d[url] = matcher
mtr.stop()
mtr.report()
return d
def GeoClassify(expdir):
# classify tweets based on lat/long context
saver.restore(session, os.path.join(expdir, 'model.bin'))
print 'preparing dataset'
dataset.Prepare(vocab, context_vocabs)
classes = [
{'lat': 40.4, 'lon': -3.7}, #madrid
{'lat': 51.5, 'lon': -0.23}, # london
{'lat': 40.7, 'lon': -74.0}, # nyc
{'lat': 41.4, 'lon': 2.17}, # barcelona
{'lat': 34.05, 'lon': -118.2}, # los angeles
{'lat': 53.5, 'lon': -2.2} # manchester
]
names = ['madrid', 'london', 'nyc', 'barcelona', 'la', 'manchester']
results = []
all_labels = []
all_preds = []
for pos in xrange(dataset.GetNumBatches()):
if pos % 10 == 0:
print pos
batch = dataset.GetNextBatch()
feed_dict = GetFeedDict(batch, use_dropout=False)
labels = []
for lat, lon in zip(feed_dict[model.context_placeholders['lat']],
feed_dict[model.context_placeholders['lon']]):
closest_dist = 300
closest_class = -1
for i in range(len(names)):
d = helper.haversine(lon, lat, classes[i]['lon'], classes[i]['lat'])
if d < closest_dist:
closest_dist = d
closest_class = names[i]
labels.append(closest_class)
labels = np.array(labels)
costs = []
for i in range(len(names)):
feed_dict[model.context_placeholders['lat']][:] = classes[i]['lat']
feed_dict[model.context_placeholders['lon']][:] = classes[i]['lon']
costs.append(session.run(model.per_sentence_loss, feed_dict))
costs = np.array(costs)
predictions = np.argmin(costs, 0)
for p, l in zip(np.squeeze(predictions), labels):
if l != '-1':
all_preds.append(p)
all_labels.append(l)
metrics.Metrics([names[i] for i in all_preds], all_labels)
def train_and_measure(data, labels, k):
model = KMeans(k)
model.fit(data, max_iter=50, visualize=False)
predictions = model.predict(data)
metric = mtr.Metrics()
wc = metric.wc_cluster_distances(data, predictions,
model.cluster_centroids)
start = time.time()
sc = metric.silhouette_coefficient(data, predictions)
# print("seconds %s" % (time.time() - start))
nmi = metric.nmi_gain(data, labels, predictions)
return model, wc, sc, nmi
def __init__(self, topic):
self.consumer = KafkaConsumer(
topic,
bootstrap_servers=['localhost:9092'],
auto_offset_reset='earliest',
enable_auto_commit=True,
auto_commit_interval_ms=5000,
group_id=None,
value_deserializer=lambda x: loads(x.decode('utf-8')))
self.sentiment_analyzer = Sentiment()
self.metrics = metrics.Metrics(consumer=self.consumer)
size = 100
self.x_vec = np.linspace(0, 1, size + 1)[0:-1]
self.y_vec = np.zeros(len(self.x_vec))
self.line1 = []
def run(self, timeseries):
print('Neptune insights starting...')
print('Listening for http requests on port 5000...')
self.webserver = WebServer(self)
self.webserver.start(5000)
print('Connecting to timeseries at: ' + timeseries + '...')
self.influx = influxdb_client.InfluxDBClient(
url=timeseries,
token=
'Ku-vr2Vu70U47XRsUhNBRB2LoCkoSAQNEEzFc8Mncw72MLvQwaQf6ct0QERwzbN7Mhy8F16apCkkR5Obg0zhaw==',
org='neptune')
self.metrics = metrics.Metrics(self.influx)
print('Neptune insights is started.')
return self
def __init__(self,
model,
scaler,
flags,
save_path,
metrics=metrics.Metrics()):
self.model_wrapper = ModelWrapper(model)
self.model_checkpoint_path = flags.save_dir
self.scaler = scaler
self.lr = flags.learning_rate
self.write_summary = flags.write_summary
self.encoder_dim = flags.encoder_dim
self.decoder_dim = flags.decoder_dim
self.batch_size = flags.batch_size
self.metrics = metrics
self.save_path = save_path
logging.get_absl_logger().addHandler(logging_base.StreamHandler())
return
def debug_mixture_classifier(opts,
step,
probs,
points,
num_plot=320,
real=True):
"""Small debugger for the mixture classifier's output.
"""
num = len(points)
if len(probs) != num:
return
if num < 2 * num_plot:
return
sorted_vals_and_ids = sorted(zip(probs, range(num)))
if real:
correct = sorted_vals_and_ids[-num_plot:]
wrong = sorted_vals_and_ids[:num_plot]
else:
correct = sorted_vals_and_ids[:num_plot]
wrong = sorted_vals_and_ids[-num_plot:]
correct_ids = [_id for val, _id in correct]
wrong_ids = [_id for val, _id in wrong]
idstring = 'real' if real else 'fake'
logging.debug('Correctly classified %s points probs:' %\
idstring)
logging.debug([val[0] for val, _id in correct])
logging.debug('Incorrectly classified %s points probs:' %\
idstring)
logging.debug([val[0] for val, _id in wrong])
metrics = metrics_lib.Metrics()
metrics.make_plots(opts,
step,
None,
points[correct_ids],
prefix='c_%s_correct_' % idstring)
metrics.make_plots(opts,
step,
None,
points[wrong_ids],
prefix='c_%s_wrong_' % idstring)
def T1(ds1_path, ds2_path, year, k):
year_graph = gr.YearGraph(ds1_path, ds2_path, year)
year_graph.read()
year_graph.weights_fraction_cocitations()
year_graph.build_ds1_graph()
metrics = me.Metrics(year_graph)
metrics.metrics_preprocessing()
metrics.pagerank(k)
models = sp.SpreadOfInfluenceModels(year_graph, metrics.get_top_k_nodes())
models.spread_preprocessing()
models.threshold_deg()
models.tipping_model()
ov_top = ov.OverlapTopics(year_graph, models.get_influenced_nodes(), k)
ov_top.overlap_preprocessing()
ov_top.clique_percolation_method()
ov_top.check_number_topics()
ov_top.check_spread_topics()
return ov_top.get_topics()
def __init__(self,
parameter=params.Params(),
network_name="CVANet_{0}".format(int(time.time())),
root_dir='experiments',
experiment_series='',
pretrained_network=''):
""" Default initialisation function.
@param parameter: An parameter object containing all relevant training parameters.
@param network_name: The name of the network to be trained.
@param root_dir: Root directory used for every output produced during training.
@param experiment_series: Path relative to the root dir used for every output produced during training.
@param pretrained_network: Path to a file containing pretrained model weights.
"""
super(Train, self).__init__(parameter=parameter,
network_name=network_name,
root_dir=root_dir,
experiment_series=experiment_series,
pretrained_network=pretrained_network)
# ---------------------------
# Initialise optimiser and metrics
# ---------------------------
self.metrics_object = metrics.Metrics(
basic_loss=parameter.loss_type,
pos_class_weight=parameter.pos_class_weight,
gmm_loss_weight=parameter.gmm_loss_weight,
geometry_loss_weight=parameter.geometry_loss_weight,
using_weighted_loss=parameter.using_weighted_loss,
weighting_loss_func=parameter.weighting_loss_func)
self.optimizer = tf.keras.optimizers.Adam(lr=parameter.learning_rate)
if parameter.task_type == 'Classification':
self.metric_names = ['Loss', 'Accuracy']
metric_formats = ['{:.3f}', '{:.3f}']
else:
self.metric_names = ['Loss']
metric_formats = ['{:.3f}']
self.metric_tracking.add_metrics(self.metric_names, metric_formats)
def main():
#get user options/configurations
opt = opts.get_opt()
#set cuda visible devices if user specified a value
if opt.gpus is not None:
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus
#load class to save metrics, iamges and models to disk
output = outputs.Outputs(opt)
#load class to store metrics and losses values
metric = metrics.Metrics()
#load synthetic dataset
healthy_dataloader_train, anomaly_dataloader_train = init_synth_dataloader_original(
opt.folder_toy_dataset, opt.batch_size, mode='train')
healthy_dataloader_val, anomaly_dataloader_val = init_synth_dataloader_original(
opt.folder_toy_dataset, opt.batch_size, mode='val')
net_g, net_r = init_model(opt)
optim_g, optim_r = init_optimizer(opt, net_g=net_g, net_r=net_r)
net_g = net_g.cuda()
net_r = net_r.cuda()
train(opt,
healthy_dataloader_train,
anomaly_dataloader_train,
healthy_dataloader_val,
anomaly_dataloader_val,
net_g=net_g,
net_r=net_r,
optim_g=optim_g,
optim_r=optim_r,
output=output,
metric=metric)
def __init__(self,
config,
agg_type,
debug=False,
verbose=False,
profile=None,
ignore_nosec=False):
'''Get logger, config, AST handler, and result store ready
:param config: config options object
:type config: bandit.core.BanditConfig
:param agg_type: aggregation type
:param debug: Whether to show debug messages or not
:param verbose: Whether to show verbose output
:param profile_name: Optional name of profile to use (from cmd line)
:param ignore_nosec: Whether to ignore #nosec or not
:return:
'''
self.debug = debug
self.verbose = verbose
if not profile:
profile = {}
self.ignore_nosec = ignore_nosec
self.b_conf = config
self.files_list = []
self.excluded_files = []
self.b_ma = b_meta_ast.BanditMetaAst()
self.skipped = []
self.results = []
self.baseline = []
self.agg_type = agg_type
self.metrics = metrics.Metrics()
self.b_ts = b_test_set.BanditTestSet(config, profile)
# set the increment of after how many files to show progress
self.progress = b_constants.progress_increment
self.scores = []
def get_current_network():
network_timestamp = request.args.get('timestamp')
if(network_timestamp is None):
curr_object = metrics.Metrics()
return jsonify(curr_object.get_network())
def main():
parser = argparse.ArgumentParser(
description='Run simulation on JSON file.')
parser.add_argument('--json',
'-j',
action='store',
dest='json_file_name',
help='Use network stored in json file',
required=True)
#option for tcp reno or tcp fast
tcp_type = parser.add_mutually_exclusive_group(required=True)
tcp_type.add_argument('--Reno',
dest='tcp_type',
action='store_const',
const='Reno',
help='Use the TCP-Reno congestion control algorithm')
tcp_type.add_argument("--FAST",
dest='tcp_type',
action='store_const',
const='FAST',
help='Use the TCP-FAST congestion control algorithm')
# options for graphing metrics
metrics = parser.add_argument_group()
metrics.add_argument('-m',
dest='metrics',
action='store_true',
help='Print graphs for metrics.\
Requires the following subarguments:')
metricType = metrics.add_mutually_exclusive_group()
metricType.add_argument('--more',
dest='log',
action='store_const',
const='more',
help='Prints a timetrace from collecting\
all data.\
Requires the -m argument.')
metricType.add_argument('--less',
dest='log',
action='store_const',
const='less',
help='Prints a timetrace from collecting\
a single datum per discrete time interval. \
Subargument for the -m argument.')
metricType.add_argument('--avg',
dest='log',
action='store_const',
const='avg',
help='Prints an approximate (average) timetrace\
by collecting data over a discrete time interval. \
Subargument for the -m argument.')
metrics.add_argument('-l',
'--links',
nargs='+',
type=str,
action='store',
dest='links',
metavar='LinkID',
help='Specify which\
links are to be logged. LinkID must given in the form\
\'L1\', \'L2\', etc. Subargument for the -m argument.')
metrics.add_argument('-f',
'--flows',
nargs='+',
type=str,
action='store',
dest='flows',
metavar='FlowID',
help='Specify which\
flows are to be logged. FlowID must given in the form\
\'F1\', \'F2\', etc. Subargument for the -m argument.')
parser.add_argument('-v',
action='store_true',
dest='verbose',
help='verbose: prints out information about events,\
event time, and number of elements in priority queue')
args = parser.parse_args()
# All subargs must be present if --m is invoked
if not args.metrics and (args.log is not None or args.links is not None
or args.flows is not None):
parser.print_usage()
print "Error: -m argument is required."
return
# All subargs must be present if --m is invoked
elif args.metrics and (args.log is None or args.links is None
or args.flows is None):
parser.print_usage()
print "Error: All of --m's subargments required."
return
f = open(args.json_file_name)
parsed_data = json.loads(f.read())
if args.verbose:
print "JSON DATA:"
pprint.pprint(parsed_data)
devices = {}
links = {}
flows = {}
print "\n\n"
# Parse json data into data structures
print "Iterating over hosts:"
for host_name in parsed_data['hosts']:
print "Host ", host_name, "has data: ", parsed_data['hosts'][host_name]
host = classes.Host(str(host_name))
devices[str(host_name)] = host
print "Iterating over routers:"
for router_name in parsed_data['routers']:
print "Router ", router_name, "has data: ", parsed_data['routers'][
router_name]
router = classes.Router(str(router_name))
devices[str(router_name)] = router
print "Hosts and routers instantiated. ", "\n\n"
print "Iterating over links and adding to hosts/routers:"
for link_name in parsed_data['links']:
link_data = parsed_data['links'][link_name]
print "Link ", link_name, "has data: ", link_data
link = classes.Link(str(link_name), link_data['link_rate'],
link_data['link_delay'], link_data['link_buffer'],
devices[link_data['devices'][0]],
devices[link_data['devices'][1]])
links[str(link_name)] = link
print "Links instantiated.", "\n\n"
print "Iterating over flows:"
for flow_name in parsed_data['flows']:
flow_data = parsed_data['flows'][flow_name]
print "Flow ", flow_name, "has data: ", flow_data
flow = classes.Flow(str(flow_name), devices[flow_data['flow_src']],
devices[flow_data['flow_dest']],
flow_data['data_amt'], flow_data['flow_start'],
flow_data['theoRTT'])
flows[str(flow_name)] = flow
print "Flows instantiated.", "\n\n"
# Verifying metric inputs from command line are correct
if args.metrics:
for flowID in args.flows:
if flowID not in flows.keys():
print "Bad flowID in argument list."
return
for linkID in args.links:
if linkID not in links.keys():
print "Bad linkID in argument list."
return
network = classes.Network(devices, links, flows)
met = None
if args.metrics:
met = m.Metrics(args.log, args.flows, args.links)
simulator = simulation.Simulator(network, args.tcp_type, met)
# Generate initial routing table
print "Running..."
if args.verbose:
print "Static routing:"
simulator.staticRouting()
while not simulator.q.empty():
result = simulator.processEvent()
if args.verbose:
print "processing one event\n" + str(result)
if args.verbose:
print "------------NETWORK------------"
print "----------DEVICE DETAILS----------"
for device_name in devices:
print devices[device_name]
print "----------LINK DETAILS----------"
for link_name in links:
print links[link_name]
print "----------FLOW DETAILS----------"
for flow_name in flows:
print flows[flow_name]
print "----------STARTING SIMULATION------------"
# Flows begin:
for flow_name in flows:
flow = flows[flow_name]
counter = 0
timer = flow.flow_start
newGenEvent = simulation.Event(None, None, "INITIALIZEFLOW", timer,
flow)
simulator.insertEvent(newGenEvent)
newDynamicRoutingEvent = simulation.Event(None, None, "REROUT",
constants.REROUT_TIME, None)
simulator.insertEvent(newDynamicRoutingEvent)
while not simulator.q.empty():
result = simulator.processEvent()
if args.verbose:
print "QUEUE SIZE: " + str(
simulator.q.qsize()) + "\n" + str(result)
for flow_name in flows:
flow = flows[flow_name]
print "DATA ACKNOWLEDGED: " + str(flow.data_acknowledged)
print "DATA MADE: " + str(flow.data_amt)
print "Simulation for ", args.json_file_name[:
-4], args.tcp_type, args.log, " done!"
simulator.done()
def __init__(self, iterations, repetitions, time_limit=None):
self.iterations = iterations
self.repetitions = repetitions
self.time_limit = time_limit
self.m_metrics = metrics.Metrics()
self.mcts_instance = mcts.Mcts(self.m_metrics)
print("Initializing matchers")
matchers = init_matchers(nlp_model, database_df, destination_url_list)
print("Matchers initialized")
target = partial(get_match_df_from_single_url, nlp_model, matchers,
destination_url_list, input_class)
proc_num = mp.cpu_count() - 1
with mp.Pool(proc_num) as p:
results = p.map(target, source_url_list)
return results
if __name__ == '__main__':
mo = m.Metrics('overall')
mo.start()
lang_model = load_nlp_model("pl_core_news_sm")
url_list = load_url_list("Input/url_list3.txt")
df_phrases = prepare_input_phrases_with_lemmas(lang_model,
"Input/Morele_ahrefs.xlsx")
df_results = create_inlinks_report(lang_model,
url_list,
df_phrases,
input_class='single-news-container')
df = pd.concat(df_results)
df.reset_index(drop=True, inplace=True)
write_df_to_excel("Output/raporcik.xlsx", df)
mo.stop()
def hour_job():
print('Hourly performance metrics are stored')
metrics.Metrics().set_day_aggregate()
metrics.Metrics().get_hour_metrics()
def minute_job():
print('Performance metrics are stored')
metrics.Metrics().get_minute_metrics()
def get_current_memory():
memory_timestamp = request.args.get('timestamp')
if(memory_timestamp is None):
curr_object = metrics.Metrics()
return jsonify(curr_object.get_memory())
"Must to specify the save flag with the output path: -s.")
if results.out:
output_sample_path = results.out
#Save file flag
if results.s:
save_logs = True
#Building graph, with weights calculation strategy or not
year_graph = gr.YearGraph(ds1_path, ds2_path, y)
year_graph.read()
year_graph.weights_fraction_cocitations()
year_graph.build_ds1_graph()
metrics = me.Metrics(year_graph)
metrics.metrics_preprocessing()
metrics.pagerank(k)
models = sp.SpreadOfInfluenceModels(year_graph, metrics.get_top_k_nodes())
models.spread_preprocessing()
if t == 0:
models.threshold_deg()
threshold = "degree"
elif t == 1:
models.threshold_half()
threshold = "half"
else:
def get_current_disk():
disk_timestamp = request.args.get('timestamp')
if(disk_timestamp is None):
curr_object = metrics.Metrics()
return jsonify(curr_object.get_disk_usage())
