def check_split_tolerance(total_item_cnt, chosen_item_cnt, desired_perc,
perc_tolerance, where):
actual_perc = sf.percent_of_part(chosen_item_cnt, total_item_cnt)
perc_deviation = abs(actual_perc - desired_perc)
if perc_deviation > perc_tolerance:
print('Percentage deviation of {} for {}'.format(
perc_deviation, where))
def calc_centrality_fraction(attacked_nodes, full_centrality_name, file_loader, centrality_file_path):
if full_centrality_name == 'degree_centrality':
raise RuntimeError('Call calc_degree_centr_fraction instead')
elif full_centrality_name == 'degree_centrality': # redundant, already saved!
raise RuntimeError('Call calc_indegree_centr_fraction instead')
centrality_info = file_loader.fetch_json(centrality_file_path)
centr_by_node = centrality_info[full_centrality_name]
nodes_degree = 0.0
for node in attacked_nodes:
nodes_degree += centr_by_node[node]
tot_degree = centrality_info['total_' + full_centrality_name]
fraction = sf.percent_of_part(nodes_degree, tot_degree)
return fraction
def calc_centrality_fraction(attacked_nodes, full_centrality_name, file_loader,
centrality_file_path):
if full_centrality_name == 'degree_centrality':
raise RuntimeError('Call calc_degree_centr_fraction instead')
elif full_centrality_name == 'degree_centrality': # redundant, already saved!
raise RuntimeError('Call calc_indegree_centr_fraction instead')
centrality_info = file_loader.fetch_json(centrality_file_path)
centr_by_node = centrality_info[full_centrality_name]
nodes_degree = 0.0
for node in attacked_nodes:
nodes_degree += centr_by_node[node]
tot_degree = centrality_info['total_' + full_centrality_name]
fraction = sf.percent_of_part(nodes_degree, tot_degree)
return fraction
def calc_stats_on_centrality(attacked_nodes, full_centrality_name,
short_centrality_name, file_loader,
centrality_file_path):
# load file with precalculated centrality metrics
centrality_info = file_loader.fetch_json(centrality_file_path)
centr_by_node = centrality_info[full_centrality_name]
# TODO: can the fallback be applied to all cases?
try:
node_cnt = centrality_info['node_count']
except KeyError:
node_cnt = len(centr_by_node)
centr_sum = 0.0
for node in attacked_nodes:
centr_sum += centr_by_node[node]
# count how many nodes are in each quintile
quintiles = centrality_info[full_centrality_name + '_quintiles']
nodes_by_quintile = [0] * 5
for node in attacked_nodes:
node_quintile = 0
for i in range(0, 4):
if centr_by_node[node] > quintiles[i]:
node_quintile += 1
else:
break
nodes_by_quintile[node_quintile] += 1
# calc what percentage of the nodes in the graph are in each quintile
for i, q_node_cnt in enumerate(nodes_by_quintile):
nodes_by_quintile[i] = sf.percent_of_part(q_node_cnt, node_cnt)
# index statistics in a dictionary using shorter names
centr_stats = {}
stat_name = short_centrality_name + '_sum'
centr_stats[stat_name] = centr_sum
for i, val in enumerate(nodes_by_quintile):
stat_name = short_centrality_name + '_q_' + str(
i + 1) # q_1 is the first quantile
centr_stats[stat_name] = val
return centr_stats
def calc_stats_on_centrality(attacked_nodes, full_centrality_name, short_centrality_name, file_loader,
centrality_file_path):
# load file with precalculated centrality metrics
centrality_info = file_loader.fetch_json(centrality_file_path)
centr_by_node = centrality_info[full_centrality_name]
# TODO: can the fallback be applied to all cases?
try:
node_cnt = centrality_info['node_count']
except KeyError:
node_cnt = len(centr_by_node)
centr_sum = 0.0
for node in attacked_nodes:
centr_sum += centr_by_node[node]
# count how many nodes are in each quintile
quintiles = centrality_info[full_centrality_name + '_quintiles']
nodes_by_quintile = [0] * 5
for node in attacked_nodes:
node_quintile = 0
for i in range(0, 4):
if centr_by_node[node] > quintiles[i]:
node_quintile += 1
else:
break
nodes_by_quintile[node_quintile] += 1
# calc what percentage of the nodes in the graph are in each quintile
for i, q_node_cnt in enumerate(nodes_by_quintile):
nodes_by_quintile[i] = sf.percent_of_part(q_node_cnt, node_cnt)
# index statistics in a dictionary using shorter names
centr_stats = {}
stat_name = short_centrality_name + '_sum'
centr_stats[stat_name] = centr_sum
for i, val in enumerate(nodes_by_quintile):
stat_name = short_centrality_name + '_q_' + str(i + 1) # q_1 is the first quantile
centr_stats[stat_name] = val
return centr_stats
def calc_atkd_percent_by_role(G, attacked_nodes, result_key_by_role):
atkd_perc_by_role = {}
node_cnt_by_role = {}
node_roles = result_key_by_role.keys()
for node_role in node_roles:
node_cnt_by_role[node_role] = 0
atkd_node_cnt_by_role = node_cnt_by_role.copy()
for node in G.nodes():
node_role = G.node[node]['role']
node_cnt_by_role[node_role] += 1
for node in attacked_nodes:
node_role = G.node[node]['role']
atkd_node_cnt_by_role[node_role] += 1
for node_role in node_roles:
result_name = result_key_by_role[node_role]
atkd_perc_by_role[result_name] = sf.percent_of_part(
atkd_node_cnt_by_role[node_role], node_cnt_by_role[node_role])
return atkd_perc_by_role
def run(conf_fpath, floader):
global logger
logger = logging.getLogger(__name__)
logger.info('conf_fpath = {}'.format(conf_fpath))
conf_fpath = os.path.normpath(conf_fpath)
if os.path.isabs(conf_fpath) is False:
conf_fpath = os.path.abspath(conf_fpath)
if not os.path.isfile(conf_fpath):
raise ValueError('Invalid value for parameter "conf_fpath", no such file.\nPath: ' + conf_fpath)
config = ConfigParser()
config.read(conf_fpath)
global time
time = 0
# read graphml files and instantiate network graphs
seed = config.get('run_opts', 'seed')
if config.has_option('run_opts', 'save_death_cause'):
save_death_cause = config.getboolean('run_opts', 'save_death_cause')
else:
save_death_cause = False
# TODO: check if directory/files exist
netw_dir = os.path.normpath(config.get('paths', 'netw_dir'))
if os.path.isabs(netw_dir) is False:
netw_dir = os.path.abspath(netw_dir)
netw_a_fname = config.get('paths', 'netw_a_fname')
netw_a_fpath_in = os.path.join(netw_dir, netw_a_fname)
# A = nx.read_graphml(netw_a_fpath_in, node_type=str)
A = floader.fetch_graphml(netw_a_fpath_in, str)
netw_b_fname = config.get('paths', 'netw_b_fname')
netw_b_fpath_in = os.path.join(netw_dir, netw_b_fname)
# B = nx.read_graphml(netw_b_fpath_in, node_type=str)
B = floader.fetch_graphml(netw_b_fpath_in, str)
netw_inter_fname = config.get('paths', 'netw_inter_fname')
netw_inter_fpath_in = os.path.join(netw_dir, netw_inter_fname)
# I = nx.read_graphml(netw_inter_fpath_in, node_type=str)
I = floader.fetch_graphml(netw_inter_fpath_in, str)
# if the union graph is needed for this simulation, it's better to have it created in advance and just load it
if config.has_option('paths', 'netw_union_fname'):
netw_union_fname = config.get('paths', 'netw_union_fname')
netw_union_fpath_in = os.path.join(netw_dir, netw_union_fname)
# ab_union = nx.read_graphml(netw_union_fpath_in, node_type=str)
ab_union = floader.fetch_graphml(netw_union_fpath_in, str)
else:
ab_union = None
# read run options
attacked_netw = config.get('run_opts', 'attacked_netw')
attack_tactic = config.get('run_opts', 'attack_tactic')
if attack_tactic not in ['targeted', 'ith_node']:
attack_cnt = config.getint('run_opts', 'attacks')
intra_support_type = config.get('run_opts', 'intra_support_type')
if intra_support_type == 'cluster_size':
min_cluster_size = config.getint('run_opts', 'min_cluster_size')
inter_support_type = config.get('run_opts', 'inter_support_type')
if attacked_netw == A.graph['name']:
attacked_G = A
elif attacked_netw == B.graph['name']:
attacked_G = B
elif attacked_netw.lower() == 'both':
if ab_union is None:
raise ValueError('A union graph is needed, specify "netw_union_fname" and make sure the file exists')
attacked_G = ab_union
else:
raise ValueError('Invalid value for parameter "attacked_netw": ' + attacked_netw)
attacks_for_A_only = ['most_inter_used_distr_subs', 'most_intra_used_distr_subs', 'most_intra_used_transm_subs',
'most_intra_used_generators']
if attack_tactic in attacks_for_A_only and attacked_netw != A.graph['name']:
raise ValueError('Attack {} can only be used on the power network A'.format(attack_tactic))
usage_attacks = ['most_inter_used_distr_subs', 'most_intra_used_distr_subs', 'most_intra_used_transm_subs',
'most_intra_used_generators', 'most_inter_used_relays', 'most_intra_used_relays']
centrality_attacks = ['betweenness_centrality_rank', 'closeness_centrality_rank', 'indegree_centrality_rank',
'katz_centrality_rank']
if attack_tactic in usage_attacks:
secondary_sort = config.get('run_opts', 'secondary_sort')
if secondary_sort not in ['random_shuffle', 'sort_by_id']:
raise ValueError('Invalid value for parameter "secondary_sort": ' + secondary_sort)
if attack_tactic in centrality_attacks:
min_rank = config.getint('run_opts', 'min_rank')
calc_centr_on = config.get('run_opts', 'calc_centrality_on')
if calc_centr_on.lower() == 'attacked_netw':
G_for_centr = attacked_G
elif calc_centr_on.lower() == 'netw_inter':
# note that calculating centrality on the inter graph is not the same as calculating it on the union graph
G_for_centr = I
else:
raise ValueError('Invalid value for parameter "calc_centr_on": ' + calc_centr_on)
# load file with precalculated centrality metrics
centr_fname = 'node_centrality_{}.json'.format(G_for_centr.graph['name'])
centr_fpath = os.path.join(netw_dir, centr_fname)
centrality_info = floader.fetch_json(centr_fpath)
# read output paths
# run_stats is a file used to save step-by-step details about this run
results_dir = os.path.normpath(config.get('paths', 'results_dir'))
if os.path.isabs(results_dir) is False:
results_dir = os.path.abspath(results_dir)
run_stats_fname = config.get('paths', 'run_stats_fname')
run_stats_fpath = os.path.join(results_dir, run_stats_fname)
# end_stats is a file used to save a single line (row) of statistics at the end of the simulation
end_stats_fpath = os.path.normpath(config.get('paths', 'end_stats_fpath'))
if os.path.isabs(end_stats_fpath) is False:
end_stats_fpath = os.path.abspath(end_stats_fpath)
# ml_stats is similar to end_stats as it is used to write a line at the end of the simulation,
# but it gathers statistics used for machine learning
ml_stats_fpath = ''
if config.has_option('paths', 'ml_stats_fpath'):
ml_stats_fpath = os.path.normpath(config.get('paths', 'ml_stats_fpath'))
if os.path.isabs(ml_stats_fpath) is False:
ml_stats_fpath = os.path.abspath(ml_stats_fpath)
if config.has_option('run_opts', 'save_attacked_roles'):
save_attacked_roles = config.getboolean('run_opts', 'save_attacked_roles')
else:
save_attacked_roles = False
# ensure output directories exist and are empty
sf.ensure_dir_exists(results_dir)
sf.ensure_dir_exists(os.path.dirname(end_stats_fpath))
# read information used to identify this simulation run
sim_group = config.get('misc', 'sim_group')
instance = config.get('misc', 'instance')
# stability check
unstable_nodes = set()
if inter_support_type == 'node_interlink':
unstable_nodes.update(find_nodes_without_inter_links(A, I))
# elif inter_support_type == 'cluster_interlink':
# unstable_nodes.update(find_nodes_in_unsupported_clusters(A, I))
elif inter_support_type == 'realistic':
unstable_nodes.update(find_uncontrolled_pow_nodes(A, B, I))
else:
raise ValueError('Invalid value for parameter "inter_support_type": ' + inter_support_type)
if intra_support_type == 'giant_component':
unstable_nodes.update(find_nodes_not_in_giant_component(A))
elif intra_support_type == 'cluster_size':
unstable_nodes.update(find_nodes_in_smaller_clusters(A, min_cluster_size))
elif intra_support_type == 'realistic':
unstable_nodes.update(find_unpowered_substations(A))
else:
raise ValueError('Invalid value for parameter "intra_support_type": ' + intra_support_type)
if inter_support_type == 'node_interlink':
unstable_nodes.update(find_nodes_without_inter_links(B, I))
# elif inter_support_type == 'cluster_interlink':
# unstable_nodes.update(find_nodes_in_unsupported_clusters(B, I))
elif inter_support_type == 'realistic':
unstable_nodes.update(find_nodes_without_inter_links(B, I))
if intra_support_type == 'giant_component':
unstable_nodes.update(find_nodes_not_in_giant_component(B))
elif intra_support_type == 'cluster_size':
unstable_nodes.update(find_nodes_in_smaller_clusters(B, min_cluster_size))
# elif intra_support_type == 'realistic':
# unstable_nodes.update(list())
if len(unstable_nodes) > 0:
logger.debug('Time {}) {} nodes unstable before the initial attack: {}'.format(
time, len(unstable_nodes), sorted(unstable_nodes, key=sf.natural_sort_key)))
total_dead_a = 0
total_dead_b = 0
intra_sup_deaths_a = 0
intra_sup_deaths_b = 0
inter_sup_deaths_a = 0
inter_sup_deaths_b = 0
if save_death_cause is True and inter_support_type == 'realistic':
no_sup_ccs_deaths = 0
no_sup_relays_deaths = 0
no_com_path_deaths = 0
# execute simulation of failure propagation
with open(run_stats_fpath, 'wb') as run_stats_file:
run_stats = csv.DictWriter(run_stats_file, ['time', 'dead'], delimiter='\t', quoting=csv.QUOTE_MINIMAL)
run_stats.writeheader()
time += 1
# perform initial attack
if attack_tactic == 'random':
attacked_nodes = choose_random_nodes(attacked_G, attack_cnt, seed)
elif attack_tactic == 'ith_node':
node_rank = config.getint('run_opts', 'node_rank')
attacked_nodes = [pick_ith_node(attacked_G, node_rank)]
elif attack_tactic == 'targeted':
target_nodes = config.get('run_opts', 'target_nodes')
attacked_nodes = [node for node in target_nodes.split()] # split list on space
# TODO: use choose_nodes_by_rank, throw away _centrality_rank columns
elif attack_tactic == 'betweenness_centrality_rank':
ranked_nodes = centrality_info['betweenness_centrality_rank']
attacked_nodes = pick_nodes_by_rank(ranked_nodes, attack_cnt, min_rank)
elif attack_tactic == 'closeness_centrality_rank':
ranked_nodes = centrality_info['closeness_centrality_rank']
attacked_nodes = pick_nodes_by_rank(ranked_nodes, attack_cnt, min_rank)
elif attack_tactic == 'indegree_centrality_rank':
ranked_nodes = centrality_info['indegree_centrality_rank']
attacked_nodes = pick_nodes_by_rank(ranked_nodes, attack_cnt, min_rank)
elif attack_tactic == 'katz_centrality_rank':
ranked_nodes = centrality_info['katz_centrality_rank']
attacked_nodes = pick_nodes_by_rank(ranked_nodes, attack_cnt, min_rank)
elif attack_tactic == 'most_inter_used_distr_subs':
attacked_nodes = choose_most_inter_used_nodes(attacked_G, I, attack_cnt, 'distribution_substation',
secondary_sort, seed)
elif attack_tactic == 'most_intra_used_distr_subs':
attacked_nodes = choose_most_intra_used_nodes(attacked_G, attack_cnt, 'distribution_substation',
secondary_sort, seed)
elif attack_tactic == 'most_intra_used_transm_subs':
attacked_nodes = choose_most_intra_used_nodes(attacked_G, attack_cnt, 'transmission_substation',
secondary_sort, seed)
elif attack_tactic == 'most_intra_used_generators':
attacked_nodes = choose_most_intra_used_nodes(attacked_G, attack_cnt, 'generator', secondary_sort, seed)
else:
raise ValueError('Invalid value for parameter "attack_tactic": ' + attack_tactic)
attacked_nodes_a = []
attacked_nodes_b = []
for node in attacked_nodes:
node_netw = I.node[node]['network']
if node_netw == A.graph['name']:
attacked_nodes_a.append(node)
elif node_netw == B.graph['name']:
attacked_nodes_b.append(node)
else:
raise RuntimeError('Node {} network "{}" marked in inter graph is neither A nor B'.format(
node, node_netw))
node_cnt_a = A.number_of_nodes()
edge_cnt_a = A.number_of_edges()
p_atkd_a = sf.percent_of_part(len(attacked_nodes_a), node_cnt_a)
node_cnt_b = B.number_of_nodes()
edge_cnt_b = B.number_of_edges()
p_atkd_b = sf.percent_of_part(len(attacked_nodes_b), node_cnt_b)
p_atkd = sf.percent_of_part(len(attacked_nodes_a) + len(attacked_nodes_b), node_cnt_a + node_cnt_b)
if ml_stats_fpath: # if this string is not empty
# fractions of connectivity lost due to the attacks
lost_deg_a = calc_degree_centr_fraction(A, attacked_nodes_a)
lost_deg_b = calc_degree_centr_fraction(B, attacked_nodes_b)
lost_indeg_i = calc_indegree_centr_fraction(I, attacked_nodes_a + attacked_nodes_b)
centr_fpath = os.path.join(netw_dir, 'node_centrality_general.json')
lost_relay_betw = calc_centrality_fraction(attacked_nodes, 'relay_betweenness_centrality', floader,
centr_fpath)
lost_ts_betw = calc_centrality_fraction(attacked_nodes, 'transm_subst_betweenness_centrality', floader,
centr_fpath)
# Please note that this section only works for nodes created using realistic roles
if save_attacked_roles is True:
# count how many nodes there are for each role in the power network
nodes_a_by_role = {'generator': 0, 'transmission_substation': 0, 'distribution_substation': 0}
try:
for node in A.nodes():
node_role = A.node[node]['role']
nodes_a_by_role[node_role] += 1
except KeyError:
raise RuntimeError('Unrecognized node role {} in graph {}'.format(node_role, A.graph['name']))
# count how many nodes were attacked for each power role
atkd_nodes_a_by_role = {'generator': 0, 'transmission_substation': 0, 'distribution_substation': 0}
for node in attacked_nodes_a:
node_role = A.node[node]['role']
atkd_nodes_a_by_role[node_role] += 1
# calculate the percentage of nodes attacked for each power role
p_atkd_gen = sf.percent_of_part(atkd_nodes_a_by_role['generator'], nodes_a_by_role['generator'])
p_atkd_ts = sf.percent_of_part(atkd_nodes_a_by_role['transmission_substation'],
nodes_a_by_role['transmission_substation'])
p_atkd_ds = sf.percent_of_part(atkd_nodes_a_by_role['distribution_substation'],
nodes_a_by_role['distribution_substation'])
# count how many nodes there are for each role in the telecom network
nodes_b_by_role = {'relay': 0, 'controller': 0}
try:
for node in B.nodes():
node_role = B.node[node]['role']
nodes_b_by_role[node_role] += 1
except KeyError:
raise RuntimeError('Unrecognized node role {} in graph {}'.format(node_role, B.graph['name']))
# count how many nodes were attacked for each telecom role
atkd_nodes_b_by_role = {'relay': 0, 'controller': 0}
for node in attacked_nodes_b:
node_role = B.node[node]['role']
atkd_nodes_b_by_role[node_role] += 1
# calculate the percentage of nodes attacked for each telecom role
p_atkd_rel = sf.percent_of_part(atkd_nodes_b_by_role['relay'], nodes_b_by_role['relay'])
p_atkd_cc = sf.percent_of_part(atkd_nodes_b_by_role['controller'], nodes_b_by_role['controller'])
total_dead_a = len(attacked_nodes_a)
if total_dead_a > 0:
A.remove_nodes_from(attacked_nodes_a)
logger.info('Time {}) {} nodes of network {} failed because of initial attack: {}'.format(
time, total_dead_a, A.graph['name'], sorted(attacked_nodes_a, key=sf.natural_sort_key)))
total_dead_b = len(attacked_nodes_b)
if total_dead_b > 0:
B.remove_nodes_from(attacked_nodes_b)
logger.info('Time {}) {} nodes of network {} failed because of initial attack: {}'.format(
time, total_dead_b, B.graph['name'], sorted(attacked_nodes_b, key=sf.natural_sort_key)))
if total_dead_a == total_dead_b == 0:
logger.info('Time {}) No nodes were attacked'.format(time))
I.remove_nodes_from(attacked_nodes)
# save_state(time, A, B, I, results_dir)
run_stats.writerow({'time': time, 'dead': attacked_nodes})
updated = True
time += 1
# phase checks
while updated is True:
updated = False
# inter checks for network A
if inter_support_type == 'node_interlink':
unsupported_nodes_a = find_nodes_without_inter_links(A, I)
# elif inter_support_type == 'cluster_interlink':
# unsupported_nodes_a = find_nodes_in_unsupported_clusters(A, I)
elif inter_support_type == 'realistic':
unsupported_nodes_a = find_uncontrolled_pow_nodes(A, B, I, save_death_cause)
if save_death_cause is True and inter_support_type == 'realistic':
no_sup_ccs_deaths += len(unsupported_nodes_a['no_sup_ccs'])
no_sup_relays_deaths += len(unsupported_nodes_a['no_sup_relays'])
no_com_path_deaths += len(unsupported_nodes_a['no_com_path'])
temp_list = list()
# convert dictionary of lists to simple list
for node_list in unsupported_nodes_a.values():
temp_list.extend(node_list)
unsupported_nodes_a = temp_list
failed_cnt_a = len(unsupported_nodes_a)
if failed_cnt_a > 0:
logger.info('Time {}) {} nodes of network {} failed for lack of inter support: {}'.format(
time, failed_cnt_a, A.graph['name'], sorted(unsupported_nodes_a, key=sf.natural_sort_key)))
total_dead_a += failed_cnt_a
inter_sup_deaths_a += failed_cnt_a
A.remove_nodes_from(unsupported_nodes_a)
I.remove_nodes_from(unsupported_nodes_a)
updated = True
# save_state(time, A, B, I, results_dir)
run_stats.writerow({'time': time, 'dead': unsupported_nodes_a})
time += 1
# intra checks for network A
if intra_support_type == 'giant_component':
unsupported_nodes_a = find_nodes_not_in_giant_component(A)
elif intra_support_type == 'cluster_size':
unsupported_nodes_a = find_nodes_in_smaller_clusters(A, min_cluster_size)
elif intra_support_type == 'realistic':
unsupported_nodes_a = find_unpowered_substations(A)
failed_cnt_a = len(unsupported_nodes_a)
if failed_cnt_a > 0:
logger.info('Time {}) {} nodes of network {} failed for lack of intra support: {}'.format(
time, failed_cnt_a, A.graph['name'], sorted(unsupported_nodes_a, key=sf.natural_sort_key)))
total_dead_a += failed_cnt_a
intra_sup_deaths_a += failed_cnt_a
A.remove_nodes_from(unsupported_nodes_a)
I.remove_nodes_from(unsupported_nodes_a)
updated = True
# save_state(time, A, B, I, results_dir)
run_stats.writerow({'time': time, 'dead': unsupported_nodes_a})
time += 1
# inter checks for network B
if inter_support_type == 'node_interlink':
unsupported_nodes_b = find_nodes_without_inter_links(B, I)
# elif inter_support_type == 'cluster_interlink':
# unsupported_nodes_b = find_nodes_in_unsupported_clusters(B, I)
elif inter_support_type == 'realistic':
unsupported_nodes_b = find_nodes_without_inter_links(B, I)
failed_cnt_b = len(unsupported_nodes_b)
if failed_cnt_b > 0:
logger.info('Time {}) {} nodes of network {} failed for lack of inter support: {}'.format(
time, failed_cnt_b, B.graph['name'], sorted(unsupported_nodes_b, key=sf.natural_sort_key)))
total_dead_b += failed_cnt_b
inter_sup_deaths_b += failed_cnt_b
B.remove_nodes_from(unsupported_nodes_b)
I.remove_nodes_from(unsupported_nodes_b)
updated = True
# save_state(time, A, B, I, results_dir)
run_stats.writerow({'time': time, 'dead': unsupported_nodes_b})
time += 1
# intra checks for network B
if intra_support_type == 'giant_component':
unsupported_nodes_b = find_nodes_not_in_giant_component(B)
elif intra_support_type == 'cluster_size':
unsupported_nodes_b = find_nodes_in_smaller_clusters(B, min_cluster_size)
elif intra_support_type == 'realistic':
unsupported_nodes_b = list()
failed_cnt_b = len(unsupported_nodes_b)
if failed_cnt_b > 0:
logger.info('Time {}) {} nodes of network {} failed for lack of intra support: {}'.format(
time, failed_cnt_b, B.graph['name'], sorted(unsupported_nodes_b, key=sf.natural_sort_key)))
total_dead_b += failed_cnt_b
intra_sup_deaths_b += failed_cnt_b
B.remove_nodes_from(unsupported_nodes_b)
I.remove_nodes_from(unsupported_nodes_b)
updated = True
# save_state(time, A, B, I, results_dir)
run_stats.writerow({'time': time, 'dead': unsupported_nodes_b})
time += 1
# save_state('final', A, B, I, results_dir)
# write statistics about the final result
if os.path.isfile(end_stats_fpath) is False:
write_header = True # if we are going to create a new file, then add the header
else:
write_header = False
with open(end_stats_fpath, 'ab') as end_stats_file:
end_stats_header = ['sim_group', 'instance', '#dead', '#dead_a', '#dead_b']
if save_attacked_roles is True:
end_stats_header.extend(['#atkd_gen', '#atkd_ts', '#atkd_ds', '#atkd_rel', '#atkd_cc'])
if save_death_cause is True:
end_stats_header.extend(['no_intra_sup_a', 'no_inter_sup_a',
'no_intra_sup_b', 'no_inter_sup_b'])
if inter_support_type == 'realistic':
end_stats_header.extend(['no_sup_ccs', 'no_sup_relays', 'no_com_path'])
end_stats = csv.DictWriter(end_stats_file, end_stats_header, delimiter='\t', quoting=csv.QUOTE_MINIMAL)
if write_header is True:
end_stats.writeheader()
end_stats_row = {'sim_group': sim_group, 'instance': instance, '#dead': total_dead_a + total_dead_b,
'#dead_a': total_dead_a, '#dead_b': total_dead_b}
if save_attacked_roles is True:
end_stats_row.update({'#atkd_gen': atkd_nodes_a_by_role['generator'],
'#atkd_ts': atkd_nodes_a_by_role['transmission_substation'],
'#atkd_ds': atkd_nodes_a_by_role['distribution_substation'],
'#atkd_rel': atkd_nodes_b_by_role['relay'],
'#atkd_cc': atkd_nodes_b_by_role['controller']})
if save_death_cause is True:
end_stats_row.update({'no_intra_sup_a': intra_sup_deaths_a, 'no_inter_sup_a': inter_sup_deaths_a,
'no_intra_sup_b': intra_sup_deaths_b, 'no_inter_sup_b': inter_sup_deaths_b})
if inter_support_type == 'realistic':
end_stats_row.update({'no_sup_ccs': no_sup_ccs_deaths, 'no_sup_relays': no_sup_relays_deaths,
'no_com_path': no_com_path_deaths})
end_stats.writerow(end_stats_row)
if ml_stats_fpath: # if this string is not empty
# percentages of dead nodes over the initial number of nodes in the graph
p_dead_a = sf.percent_of_part(total_dead_a, node_cnt_a)
p_dead_b = sf.percent_of_part(total_dead_b, node_cnt_b)
p_dead = sf.percent_of_part(total_dead_a + total_dead_b, node_cnt_a + node_cnt_b)
# calculate the overall centrality of the attacked nodes
all_centr_stats = {}
centr_name = 'betweenness_centrality'
centr_fpath_a = os.path.join(netw_dir, 'node_centrality_{}.json'.format(A.graph['name']))
centr_stats = calc_stats_on_centrality(attacked_nodes_a, centr_name, 'betw_c_a', floader, centr_fpath_a)
all_centr_stats.update(centr_stats)
centr_fpath_b = os.path.join(netw_dir, 'node_centrality_{}.json'.format(B.graph['name']))
centr_stats = calc_stats_on_centrality(attacked_nodes_b, centr_name, 'betw_c_b', floader, centr_fpath_b)
all_centr_stats.update(centr_stats)
tot_node_cnt = node_cnt_a + node_cnt_b
centr_fpath_ab = os.path.join(netw_dir, 'node_centrality_{}.json'.format(ab_union.graph['name']))
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name, 'betw_c_ab', floader, centr_fpath_ab)
all_centr_stats.update(centr_stats)
centr_fpath_i = os.path.join(netw_dir, 'node_centrality_{}.json'.format(I.graph['name']))
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name, 'betw_c_i', floader, centr_fpath_i)
all_centr_stats.update(centr_stats)
centr_name = 'closeness_centrality'
centr_stats = calc_stats_on_centrality(attacked_nodes_a, centr_name, 'clos_c_a', floader, centr_fpath_a)
all_centr_stats.update(centr_stats)
centr_stats = calc_stats_on_centrality(attacked_nodes_b, centr_name, 'clos_c_b', floader, centr_fpath_b)
all_centr_stats.update(centr_stats)
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name, 'clos_c_ab', floader, centr_fpath_ab)
all_centr_stats.update(centr_stats)
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name, 'clos_c_i', floader, centr_fpath_i)
all_centr_stats.update(centr_stats)
centr_name = 'indegree_centrality'
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name, 'indeg_c_ab', floader, centr_fpath_ab)
all_centr_stats.update(centr_stats)
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name, 'indeg_c_i', floader, centr_fpath_i)
all_centr_stats.update(centr_stats)
centr_name = 'katz_centrality'
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name, 'katz_c_ab', floader, centr_fpath_ab)
all_centr_stats.update(centr_stats)
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name, 'katz_c_i', floader, centr_fpath_i)
all_centr_stats.update(centr_stats)
centr_name = 'relay_betweenness_centrality'
centr_fpath_gen = os.path.join(netw_dir, 'node_centrality_general.json')
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name, 'rel_betw_c', floader, centr_fpath_gen)
all_centr_stats.update(centr_stats)
centr_name = 'transm_subst_betweenness_centrality'
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name, 'ts_betw_c', floader, centr_fpath_gen)
all_centr_stats.update(centr_stats)
# if we need to create a new file, then remember to add the header
if os.path.isfile(ml_stats_fpath) is False:
write_header = True
else:
write_header = False
with open(ml_stats_fpath, 'ab') as ml_stats_file:
ml_stats_header = ['sim_group', 'instance', '#nodes_a', '#edges_a', '#nodes_b', '#edges_b',
'p_atkd', 'p_dead', 'p_atkd_a', 'p_dead_a', 'p_atkd_b', 'p_dead_b']
# add statistics about centrality, sorted so they can be more found easily in the output file
ml_stats_header.extend(sorted(all_centr_stats.keys(), key=sf.natural_sort_key))
if save_attacked_roles is True:
ml_stats_header.extend(['p_atkd_gen', 'p_atkd_ts', 'p_atkd_ds', 'p_atkd_rel', 'p_atkd_cc'])
ml_stats_header.extend(['lost_deg_a', 'lost_deg_b', 'lost_indeg_i', 'lost_rel_betw', 'lost_ts_betw'])
if save_death_cause is True:
ml_stats_header.extend(['p_no_intra_sup_a', 'p_no_inter_sup_a',
'p_no_intra_sup_b', 'p_no_inter_sup_b'])
if inter_support_type == 'realistic':
ml_stats_header.extend(['p_no_sup_ccs', 'p_no_sup_relays', 'p_no_com_path'])
ml_stats = csv.DictWriter(ml_stats_file, ml_stats_header, delimiter='\t', quoting=csv.QUOTE_MINIMAL)
if write_header is True:
ml_stats.writeheader()
ml_stats_row = {'sim_group': sim_group, 'instance': instance,
'#nodes_a': node_cnt_a, '#edges_a': edge_cnt_a,
'#nodes_b': node_cnt_b, '#edges_b': edge_cnt_b,
'p_atkd': p_atkd, 'p_atkd_a': p_atkd_a, 'p_atkd_b': p_atkd_b,
'p_dead': p_dead, 'p_dead_a': p_dead_a, 'p_dead_b': p_dead_b}
ml_stats_row.update(all_centr_stats)
if save_attacked_roles is True:
ml_stats_row.update({'p_atkd_gen': p_atkd_gen, 'p_atkd_ts': p_atkd_ts, 'p_atkd_ds': p_atkd_ds,
'p_atkd_rel': p_atkd_rel, 'p_atkd_cc': p_atkd_cc,})
ml_stats_row.update({'lost_deg_a': lost_deg_a, 'lost_deg_b': lost_deg_b, 'lost_indeg_i': lost_indeg_i,
'lost_rel_betw': lost_relay_betw, 'lost_ts_betw': lost_ts_betw})
if save_death_cause is True:
p_no_intra_sup_a = sf.percent_of_part(intra_sup_deaths_a, total_dead_a)
p_no_inter_sup_a = sf.percent_of_part(inter_sup_deaths_a, total_dead_a)
p_no_intra_sup_b = sf.percent_of_part(intra_sup_deaths_b, total_dead_b)
p_no_inter_sup_b = sf.percent_of_part(inter_sup_deaths_b, total_dead_b)
ml_stats_row.update({'p_no_intra_sup_a': p_no_intra_sup_a, 'p_no_inter_sup_a': p_no_inter_sup_a,
'p_no_intra_sup_b': p_no_intra_sup_b, 'p_no_inter_sup_b': p_no_inter_sup_b})
if inter_support_type == 'realistic':
p_no_sup_ccs = sf.percent_of_part(no_sup_ccs_deaths, total_dead_a)
p_no_sup_relays = sf.percent_of_part(no_sup_relays_deaths, total_dead_a)
p_no_com_path = sf.percent_of_part(no_com_path_deaths, total_dead_a)
ml_stats_row.update({'p_no_sup_ccs': p_no_sup_ccs, 'p_no_sup_relays': p_no_sup_relays,
'p_no_com_path': p_no_com_path})
ml_stats.writerow(ml_stats_row)
def calc_indegree_centr_fraction(G, nodes):
tot_indegree = sum(G.in_degree(G.nodes()).values())
nodes_indegree = sum(G.in_degree(nodes).values())
fraction = sf.percent_of_part(nodes_indegree, tot_indegree)
return fraction
def run(conf_fpath, floader):
global logger
logger = logging.getLogger(__name__)
logger.info('conf_fpath = {}'.format(conf_fpath))
conf_fpath = os.path.normpath(conf_fpath)
if os.path.isabs(conf_fpath) is False:
conf_fpath = os.path.abspath(conf_fpath)
if not os.path.isfile(conf_fpath):
raise ValueError(
'Invalid value for parameter "conf_fpath", no such file.\nPath: ' +
conf_fpath)
config = ConfigParser()
config.read(conf_fpath)
global time
time = 0
# read graphml files and instantiate network graphs
seed = config.get('run_opts', 'seed')
if config.has_option('run_opts', 'save_death_cause'):
save_death_cause = config.getboolean('run_opts', 'save_death_cause')
else:
save_death_cause = False
# TODO: check if directory/files exist
netw_dir = os.path.normpath(config.get('paths', 'netw_dir'))
if os.path.isabs(netw_dir) is False:
netw_dir = os.path.abspath(netw_dir)
netw_a_fname = config.get('paths', 'netw_a_fname')
netw_a_fpath_in = os.path.join(netw_dir, netw_a_fname)
# A = nx.read_graphml(netw_a_fpath_in, node_type=str)
A = floader.fetch_graphml(netw_a_fpath_in, str)
netw_b_fname = config.get('paths', 'netw_b_fname')
netw_b_fpath_in = os.path.join(netw_dir, netw_b_fname)
# B = nx.read_graphml(netw_b_fpath_in, node_type=str)
B = floader.fetch_graphml(netw_b_fpath_in, str)
netw_inter_fname = config.get('paths', 'netw_inter_fname')
netw_inter_fpath_in = os.path.join(netw_dir, netw_inter_fname)
# I = nx.read_graphml(netw_inter_fpath_in, node_type=str)
I = floader.fetch_graphml(netw_inter_fpath_in, str)
# if the union graph is needed for this simulation, it's better to have it created in advance and just load it
if config.has_option('paths', 'netw_union_fname'):
netw_union_fname = config.get('paths', 'netw_union_fname')
netw_union_fpath_in = os.path.join(netw_dir, netw_union_fname)
# ab_union = nx.read_graphml(netw_union_fpath_in, node_type=str)
ab_union = floader.fetch_graphml(netw_union_fpath_in, str)
else:
ab_union = None
# read run options
attacked_netw = config.get('run_opts', 'attacked_netw')
attack_tactic = config.get('run_opts', 'attack_tactic')
if attack_tactic not in ['targeted', 'ith_node']:
attack_cnt = config.getint('run_opts', 'attacks')
intra_support_type = config.get('run_opts', 'intra_support_type')
if intra_support_type == 'cluster_size':
min_cluster_size = config.getint('run_opts', 'min_cluster_size')
inter_support_type = config.get('run_opts', 'inter_support_type')
if attacked_netw == A.graph['name']:
attacked_G = A
elif attacked_netw == B.graph['name']:
attacked_G = B
elif attacked_netw.lower() == 'both':
if ab_union is None:
raise ValueError(
'A union graph is needed, specify "netw_union_fname" and make sure the file exists'
)
attacked_G = ab_union
else:
raise ValueError('Invalid value for parameter "attacked_netw": ' +
attacked_netw)
attacks_for_A_only = [
'most_inter_used_distr_subs', 'most_intra_used_distr_subs',
'most_intra_used_transm_subs', 'most_intra_used_generators'
]
if attack_tactic in attacks_for_A_only and attacked_netw != A.graph['name']:
raise ValueError(
'Attack {} can only be used on the power network A'.format(
attack_tactic))
usage_attacks = [
'most_inter_used_distr_subs', 'most_intra_used_distr_subs',
'most_intra_used_transm_subs', 'most_intra_used_generators',
'most_inter_used_relays', 'most_intra_used_relays'
]
centrality_attacks = [
'betweenness_centrality_rank', 'closeness_centrality_rank',
'indegree_centrality_rank', 'katz_centrality_rank'
]
if attack_tactic in usage_attacks:
secondary_sort = config.get('run_opts', 'secondary_sort')
if secondary_sort not in ['random_shuffle', 'sort_by_id']:
raise ValueError('Invalid value for parameter "secondary_sort": ' +
secondary_sort)
if attack_tactic in centrality_attacks:
min_rank = config.getint('run_opts', 'min_rank')
calc_centr_on = config.get('run_opts', 'calc_centrality_on')
if calc_centr_on.lower() == 'attacked_netw':
G_for_centr = attacked_G
elif calc_centr_on.lower() == 'netw_inter':
# note that calculating centrality on the inter graph is not the same as calculating it on the union graph
G_for_centr = I
else:
raise ValueError('Invalid value for parameter "calc_centr_on": ' +
calc_centr_on)
# load file with precalculated centrality metrics
centr_fname = 'node_centrality_{}.json'.format(
G_for_centr.graph['name'])
centr_fpath = os.path.join(netw_dir, centr_fname)
centrality_info = floader.fetch_json(centr_fpath)
# read output paths
# run_stats is a file used to save step-by-step details about this run
results_dir = os.path.normpath(config.get('paths', 'results_dir'))
if os.path.isabs(results_dir) is False:
results_dir = os.path.abspath(results_dir)
run_stats_fname = config.get('paths', 'run_stats_fname')
run_stats_fpath = os.path.join(results_dir, run_stats_fname)
# end_stats is a file used to save a single line (row) of statistics at the end of the simulation
end_stats_fpath = os.path.normpath(config.get('paths', 'end_stats_fpath'))
if os.path.isabs(end_stats_fpath) is False:
end_stats_fpath = os.path.abspath(end_stats_fpath)
# ml_stats is similar to end_stats as it is used to write a line at the end of the simulation,
# but it gathers statistics used for machine learning
ml_stats_fpath = ''
if config.has_option('paths', 'ml_stats_fpath'):
ml_stats_fpath = os.path.normpath(config.get('paths',
'ml_stats_fpath'))
if os.path.isabs(ml_stats_fpath) is False:
ml_stats_fpath = os.path.abspath(ml_stats_fpath)
if config.has_option('run_opts', 'save_attacked_roles'):
save_attacked_roles = config.getboolean('run_opts',
'save_attacked_roles')
else:
save_attacked_roles = False
# ensure output directories exist and are empty
sf.ensure_dir_exists(results_dir)
sf.ensure_dir_exists(os.path.dirname(end_stats_fpath))
# read information used to identify this simulation run
sim_group = config.get('misc', 'sim_group')
instance = config.get('misc', 'instance')
# stability check
unstable_nodes = set()
if inter_support_type == 'node_interlink':
unstable_nodes.update(find_nodes_without_inter_links(A, I))
# elif inter_support_type == 'cluster_interlink':
# unstable_nodes.update(find_nodes_in_unsupported_clusters(A, I))
elif inter_support_type == 'realistic':
unstable_nodes.update(find_uncontrolled_pow_nodes(A, B, I))
else:
raise ValueError('Invalid value for parameter "inter_support_type": ' +
inter_support_type)
if intra_support_type == 'giant_component':
unstable_nodes.update(find_nodes_not_in_giant_component(A))
elif intra_support_type == 'cluster_size':
unstable_nodes.update(
find_nodes_in_smaller_clusters(A, min_cluster_size))
elif intra_support_type == 'realistic':
unstable_nodes.update(find_unpowered_substations(A))
else:
raise ValueError('Invalid value for parameter "intra_support_type": ' +
intra_support_type)
if inter_support_type == 'node_interlink':
unstable_nodes.update(find_nodes_without_inter_links(B, I))
# elif inter_support_type == 'cluster_interlink':
# unstable_nodes.update(find_nodes_in_unsupported_clusters(B, I))
elif inter_support_type == 'realistic':
unstable_nodes.update(find_nodes_without_inter_links(B, I))
if intra_support_type == 'giant_component':
unstable_nodes.update(find_nodes_not_in_giant_component(B))
elif intra_support_type == 'cluster_size':
unstable_nodes.update(
find_nodes_in_smaller_clusters(B, min_cluster_size))
# elif intra_support_type == 'realistic':
# unstable_nodes.update(list())
if len(unstable_nodes) > 0:
logger.debug(
'Time {}) {} nodes unstable before the initial attack: {}'.format(
time, len(unstable_nodes),
sorted(unstable_nodes, key=sf.natural_sort_key)))
total_dead_a = 0
total_dead_b = 0
intra_sup_deaths_a = 0
intra_sup_deaths_b = 0
inter_sup_deaths_a = 0
inter_sup_deaths_b = 0
if save_death_cause is True and inter_support_type == 'realistic':
no_sup_ccs_deaths = 0
no_sup_relays_deaths = 0
no_com_path_deaths = 0
# execute simulation of failure propagation
with open(run_stats_fpath, 'wb') as run_stats_file:
run_stats = csv.DictWriter(run_stats_file, ['time', 'dead'],
delimiter='\t',
quoting=csv.QUOTE_MINIMAL)
run_stats.writeheader()
time += 1
# perform initial attack
if attack_tactic == 'random':
attacked_nodes = choose_random_nodes(attacked_G, attack_cnt, seed)
elif attack_tactic == 'ith_node':
node_rank = config.getint('run_opts', 'node_rank')
attacked_nodes = [pick_ith_node(attacked_G, node_rank)]
elif attack_tactic == 'targeted':
target_nodes = config.get('run_opts', 'target_nodes')
attacked_nodes = [node for node in target_nodes.split()
] # split list on space
# TODO: use choose_nodes_by_rank, throw away _centrality_rank columns
elif attack_tactic == 'betweenness_centrality_rank':
ranked_nodes = centrality_info['betweenness_centrality_rank']
attacked_nodes = pick_nodes_by_rank(ranked_nodes, attack_cnt,
min_rank)
elif attack_tactic == 'closeness_centrality_rank':
ranked_nodes = centrality_info['closeness_centrality_rank']
attacked_nodes = pick_nodes_by_rank(ranked_nodes, attack_cnt,
min_rank)
elif attack_tactic == 'indegree_centrality_rank':
ranked_nodes = centrality_info['indegree_centrality_rank']
attacked_nodes = pick_nodes_by_rank(ranked_nodes, attack_cnt,
min_rank)
elif attack_tactic == 'katz_centrality_rank':
ranked_nodes = centrality_info['katz_centrality_rank']
attacked_nodes = pick_nodes_by_rank(ranked_nodes, attack_cnt,
min_rank)
elif attack_tactic == 'most_inter_used_distr_subs':
attacked_nodes = choose_most_inter_used_nodes(
attacked_G, I, attack_cnt, 'distribution_substation',
secondary_sort, seed)
elif attack_tactic == 'most_intra_used_distr_subs':
attacked_nodes = choose_most_intra_used_nodes(
attacked_G, attack_cnt, 'distribution_substation',
secondary_sort, seed)
elif attack_tactic == 'most_intra_used_transm_subs':
attacked_nodes = choose_most_intra_used_nodes(
attacked_G, attack_cnt, 'transmission_substation',
secondary_sort, seed)
elif attack_tactic == 'most_intra_used_generators':
attacked_nodes = choose_most_intra_used_nodes(
attacked_G, attack_cnt, 'generator', secondary_sort, seed)
else:
raise ValueError('Invalid value for parameter "attack_tactic": ' +
attack_tactic)
attacked_nodes_a = []
attacked_nodes_b = []
for node in attacked_nodes:
node_netw = I.node[node]['network']
if node_netw == A.graph['name']:
attacked_nodes_a.append(node)
elif node_netw == B.graph['name']:
attacked_nodes_b.append(node)
else:
raise RuntimeError(
'Node {} network "{}" marked in inter graph is neither A nor B'
.format(node, node_netw))
node_cnt_a = A.number_of_nodes()
edge_cnt_a = A.number_of_edges()
p_atkd_a = sf.percent_of_part(len(attacked_nodes_a), node_cnt_a)
node_cnt_b = B.number_of_nodes()
edge_cnt_b = B.number_of_edges()
p_atkd_b = sf.percent_of_part(len(attacked_nodes_b), node_cnt_b)
p_atkd = sf.percent_of_part(
len(attacked_nodes_a) + len(attacked_nodes_b),
node_cnt_a + node_cnt_b)
if ml_stats_fpath: # if this string is not empty
# fractions of connectivity lost due to the attacks
lost_deg_a = calc_degree_centr_fraction(A, attacked_nodes_a)
lost_deg_b = calc_degree_centr_fraction(B, attacked_nodes_b)
lost_indeg_i = calc_indegree_centr_fraction(
I, attacked_nodes_a + attacked_nodes_b)
centr_fpath = os.path.join(netw_dir,
'node_centrality_general.json')
lost_relay_betw = calc_centrality_fraction(
attacked_nodes, 'relay_betweenness_centrality', floader,
centr_fpath)
lost_ts_betw = calc_centrality_fraction(
attacked_nodes, 'transm_subst_betweenness_centrality', floader,
centr_fpath)
# Please note that this section only works for nodes created using realistic roles
if save_attacked_roles is True:
# count how many nodes there are for each role in the power network
nodes_a_by_role = {
'generator': 0,
'transmission_substation': 0,
'distribution_substation': 0
}
try:
for node in A.nodes():
node_role = A.node[node]['role']
nodes_a_by_role[node_role] += 1
except KeyError:
raise RuntimeError(
'Unrecognized node role {} in graph {}'.format(
node_role, A.graph['name']))
# count how many nodes were attacked for each power role
atkd_nodes_a_by_role = {
'generator': 0,
'transmission_substation': 0,
'distribution_substation': 0
}
for node in attacked_nodes_a:
node_role = A.node[node]['role']
atkd_nodes_a_by_role[node_role] += 1
# calculate the percentage of nodes attacked for each power role
p_atkd_gen = sf.percent_of_part(
atkd_nodes_a_by_role['generator'],
nodes_a_by_role['generator'])
p_atkd_ts = sf.percent_of_part(
atkd_nodes_a_by_role['transmission_substation'],
nodes_a_by_role['transmission_substation'])
p_atkd_ds = sf.percent_of_part(
atkd_nodes_a_by_role['distribution_substation'],
nodes_a_by_role['distribution_substation'])
# count how many nodes there are for each role in the telecom network
nodes_b_by_role = {'relay': 0, 'controller': 0}
try:
for node in B.nodes():
node_role = B.node[node]['role']
nodes_b_by_role[node_role] += 1
except KeyError:
raise RuntimeError(
'Unrecognized node role {} in graph {}'.format(
node_role, B.graph['name']))
# count how many nodes were attacked for each telecom role
atkd_nodes_b_by_role = {'relay': 0, 'controller': 0}
for node in attacked_nodes_b:
node_role = B.node[node]['role']
atkd_nodes_b_by_role[node_role] += 1
# calculate the percentage of nodes attacked for each telecom role
p_atkd_rel = sf.percent_of_part(atkd_nodes_b_by_role['relay'],
nodes_b_by_role['relay'])
p_atkd_cc = sf.percent_of_part(
atkd_nodes_b_by_role['controller'],
nodes_b_by_role['controller'])
total_dead_a = len(attacked_nodes_a)
if total_dead_a > 0:
A.remove_nodes_from(attacked_nodes_a)
logger.info(
'Time {}) {} nodes of network {} failed because of initial attack: {}'
.format(time, total_dead_a, A.graph['name'],
sorted(attacked_nodes_a, key=sf.natural_sort_key)))
total_dead_b = len(attacked_nodes_b)
if total_dead_b > 0:
B.remove_nodes_from(attacked_nodes_b)
logger.info(
'Time {}) {} nodes of network {} failed because of initial attack: {}'
.format(time, total_dead_b, B.graph['name'],
sorted(attacked_nodes_b, key=sf.natural_sort_key)))
if total_dead_a == total_dead_b == 0:
logger.info('Time {}) No nodes were attacked'.format(time))
I.remove_nodes_from(attacked_nodes)
# save_state(time, A, B, I, results_dir)
run_stats.writerow({'time': time, 'dead': attacked_nodes})
updated = True
time += 1
# phase checks
while updated is True:
updated = False
# inter checks for network A
if inter_support_type == 'node_interlink':
unsupported_nodes_a = find_nodes_without_inter_links(A, I)
# elif inter_support_type == 'cluster_interlink':
# unsupported_nodes_a = find_nodes_in_unsupported_clusters(A, I)
elif inter_support_type == 'realistic':
unsupported_nodes_a = find_uncontrolled_pow_nodes(
A, B, I, save_death_cause)
if save_death_cause is True and inter_support_type == 'realistic':
no_sup_ccs_deaths += len(unsupported_nodes_a['no_sup_ccs'])
no_sup_relays_deaths += len(
unsupported_nodes_a['no_sup_relays'])
no_com_path_deaths += len(unsupported_nodes_a['no_com_path'])
temp_list = list()
# convert dictionary of lists to simple list
for node_list in unsupported_nodes_a.values():
temp_list.extend(node_list)
unsupported_nodes_a = temp_list
failed_cnt_a = len(unsupported_nodes_a)
if failed_cnt_a > 0:
logger.info(
'Time {}) {} nodes of network {} failed for lack of inter support: {}'
.format(
time, failed_cnt_a, A.graph['name'],
sorted(unsupported_nodes_a, key=sf.natural_sort_key)))
total_dead_a += failed_cnt_a
inter_sup_deaths_a += failed_cnt_a
A.remove_nodes_from(unsupported_nodes_a)
I.remove_nodes_from(unsupported_nodes_a)
updated = True
# save_state(time, A, B, I, results_dir)
run_stats.writerow({'time': time, 'dead': unsupported_nodes_a})
time += 1
# intra checks for network A
if intra_support_type == 'giant_component':
unsupported_nodes_a = find_nodes_not_in_giant_component(A)
elif intra_support_type == 'cluster_size':
unsupported_nodes_a = find_nodes_in_smaller_clusters(
A, min_cluster_size)
elif intra_support_type == 'realistic':
unsupported_nodes_a = find_unpowered_substations(A)
failed_cnt_a = len(unsupported_nodes_a)
if failed_cnt_a > 0:
logger.info(
'Time {}) {} nodes of network {} failed for lack of intra support: {}'
.format(
time, failed_cnt_a, A.graph['name'],
sorted(unsupported_nodes_a, key=sf.natural_sort_key)))
total_dead_a += failed_cnt_a
intra_sup_deaths_a += failed_cnt_a
A.remove_nodes_from(unsupported_nodes_a)
I.remove_nodes_from(unsupported_nodes_a)
updated = True
# save_state(time, A, B, I, results_dir)
run_stats.writerow({'time': time, 'dead': unsupported_nodes_a})
time += 1
# inter checks for network B
if inter_support_type == 'node_interlink':
unsupported_nodes_b = find_nodes_without_inter_links(B, I)
# elif inter_support_type == 'cluster_interlink':
# unsupported_nodes_b = find_nodes_in_unsupported_clusters(B, I)
elif inter_support_type == 'realistic':
unsupported_nodes_b = find_nodes_without_inter_links(B, I)
failed_cnt_b = len(unsupported_nodes_b)
if failed_cnt_b > 0:
logger.info(
'Time {}) {} nodes of network {} failed for lack of inter support: {}'
.format(
time, failed_cnt_b, B.graph['name'],
sorted(unsupported_nodes_b, key=sf.natural_sort_key)))
total_dead_b += failed_cnt_b
inter_sup_deaths_b += failed_cnt_b
B.remove_nodes_from(unsupported_nodes_b)
I.remove_nodes_from(unsupported_nodes_b)
updated = True
# save_state(time, A, B, I, results_dir)
run_stats.writerow({'time': time, 'dead': unsupported_nodes_b})
time += 1
# intra checks for network B
if intra_support_type == 'giant_component':
unsupported_nodes_b = find_nodes_not_in_giant_component(B)
elif intra_support_type == 'cluster_size':
unsupported_nodes_b = find_nodes_in_smaller_clusters(
B, min_cluster_size)
elif intra_support_type == 'realistic':
unsupported_nodes_b = list()
failed_cnt_b = len(unsupported_nodes_b)
if failed_cnt_b > 0:
logger.info(
'Time {}) {} nodes of network {} failed for lack of intra support: {}'
.format(
time, failed_cnt_b, B.graph['name'],
sorted(unsupported_nodes_b, key=sf.natural_sort_key)))
total_dead_b += failed_cnt_b
intra_sup_deaths_b += failed_cnt_b
B.remove_nodes_from(unsupported_nodes_b)
I.remove_nodes_from(unsupported_nodes_b)
updated = True
# save_state(time, A, B, I, results_dir)
run_stats.writerow({'time': time, 'dead': unsupported_nodes_b})
time += 1
# save_state('final', A, B, I, results_dir)
# write statistics about the final result
if os.path.isfile(end_stats_fpath) is False:
write_header = True # if we are going to create a new file, then add the header
else:
write_header = False
with open(end_stats_fpath, 'ab') as end_stats_file:
end_stats_header = [
'sim_group', 'instance', '#dead', '#dead_a', '#dead_b'
]
if save_attacked_roles is True:
end_stats_header.extend(
['#atkd_gen', '#atkd_ts', '#atkd_ds', '#atkd_rel', '#atkd_cc'])
if save_death_cause is True:
end_stats_header.extend([
'no_intra_sup_a', 'no_inter_sup_a', 'no_intra_sup_b',
'no_inter_sup_b'
])
if inter_support_type == 'realistic':
end_stats_header.extend(
['no_sup_ccs', 'no_sup_relays', 'no_com_path'])
end_stats = csv.DictWriter(end_stats_file,
end_stats_header,
delimiter='\t',
quoting=csv.QUOTE_MINIMAL)
if write_header is True:
end_stats.writeheader()
end_stats_row = {
'sim_group': sim_group,
'instance': instance,
'#dead': total_dead_a + total_dead_b,
'#dead_a': total_dead_a,
'#dead_b': total_dead_b
}
if save_attacked_roles is True:
end_stats_row.update({
'#atkd_gen':
atkd_nodes_a_by_role['generator'],
'#atkd_ts':
atkd_nodes_a_by_role['transmission_substation'],
'#atkd_ds':
atkd_nodes_a_by_role['distribution_substation'],
'#atkd_rel':
atkd_nodes_b_by_role['relay'],
'#atkd_cc':
atkd_nodes_b_by_role['controller']
})
if save_death_cause is True:
end_stats_row.update({
'no_intra_sup_a': intra_sup_deaths_a,
'no_inter_sup_a': inter_sup_deaths_a,
'no_intra_sup_b': intra_sup_deaths_b,
'no_inter_sup_b': inter_sup_deaths_b
})
if inter_support_type == 'realistic':
end_stats_row.update({
'no_sup_ccs': no_sup_ccs_deaths,
'no_sup_relays': no_sup_relays_deaths,
'no_com_path': no_com_path_deaths
})
end_stats.writerow(end_stats_row)
if ml_stats_fpath: # if this string is not empty
# percentages of dead nodes over the initial number of nodes in the graph
p_dead_a = sf.percent_of_part(total_dead_a, node_cnt_a)
p_dead_b = sf.percent_of_part(total_dead_b, node_cnt_b)
p_dead = sf.percent_of_part(total_dead_a + total_dead_b,
node_cnt_a + node_cnt_b)
# calculate the overall centrality of the attacked nodes
all_centr_stats = {}
centr_name = 'betweenness_centrality'
centr_fpath_a = os.path.join(
netw_dir, 'node_centrality_{}.json'.format(A.graph['name']))
centr_stats = calc_stats_on_centrality(attacked_nodes_a, centr_name,
'betw_c_a', floader,
centr_fpath_a)
all_centr_stats.update(centr_stats)
centr_fpath_b = os.path.join(
netw_dir, 'node_centrality_{}.json'.format(B.graph['name']))
centr_stats = calc_stats_on_centrality(attacked_nodes_b, centr_name,
'betw_c_b', floader,
centr_fpath_b)
all_centr_stats.update(centr_stats)
tot_node_cnt = node_cnt_a + node_cnt_b
centr_fpath_ab = os.path.join(
netw_dir, 'node_centrality_{}.json'.format(ab_union.graph['name']))
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name,
'betw_c_ab', floader,
centr_fpath_ab)
all_centr_stats.update(centr_stats)
centr_fpath_i = os.path.join(
netw_dir, 'node_centrality_{}.json'.format(I.graph['name']))
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name,
'betw_c_i', floader,
centr_fpath_i)
all_centr_stats.update(centr_stats)
centr_name = 'closeness_centrality'
centr_stats = calc_stats_on_centrality(attacked_nodes_a, centr_name,
'clos_c_a', floader,
centr_fpath_a)
all_centr_stats.update(centr_stats)
centr_stats = calc_stats_on_centrality(attacked_nodes_b, centr_name,
'clos_c_b', floader,
centr_fpath_b)
all_centr_stats.update(centr_stats)
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name,
'clos_c_ab', floader,
centr_fpath_ab)
all_centr_stats.update(centr_stats)
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name,
'clos_c_i', floader,
centr_fpath_i)
all_centr_stats.update(centr_stats)
centr_name = 'indegree_centrality'
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name,
'indeg_c_ab', floader,
centr_fpath_ab)
all_centr_stats.update(centr_stats)
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name,
'indeg_c_i', floader,
centr_fpath_i)
all_centr_stats.update(centr_stats)
centr_name = 'katz_centrality'
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name,
'katz_c_ab', floader,
centr_fpath_ab)
all_centr_stats.update(centr_stats)
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name,
'katz_c_i', floader,
centr_fpath_i)
all_centr_stats.update(centr_stats)
centr_name = 'relay_betweenness_centrality'
centr_fpath_gen = os.path.join(netw_dir,
'node_centrality_general.json')
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name,
'rel_betw_c', floader,
centr_fpath_gen)
all_centr_stats.update(centr_stats)
centr_name = 'transm_subst_betweenness_centrality'
centr_stats = calc_stats_on_centrality(attacked_nodes, centr_name,
'ts_betw_c', floader,
centr_fpath_gen)
all_centr_stats.update(centr_stats)
# if we need to create a new file, then remember to add the header
if os.path.isfile(ml_stats_fpath) is False:
write_header = True
else:
write_header = False
with open(ml_stats_fpath, 'ab') as ml_stats_file:
ml_stats_header = [
'sim_group', 'instance', '#nodes_a', '#edges_a', '#nodes_b',
'#edges_b', 'p_atkd', 'p_dead', 'p_atkd_a', 'p_dead_a',
'p_atkd_b', 'p_dead_b'
]
# add statistics about centrality, sorted so they can be more found easily in the output file
ml_stats_header.extend(
sorted(all_centr_stats.keys(), key=sf.natural_sort_key))
if save_attacked_roles is True:
ml_stats_header.extend([
'p_atkd_gen', 'p_atkd_ts', 'p_atkd_ds', 'p_atkd_rel',
'p_atkd_cc'
])
ml_stats_header.extend([
'lost_deg_a', 'lost_deg_b', 'lost_indeg_i', 'lost_rel_betw',
'lost_ts_betw'
])
if save_death_cause is True:
ml_stats_header.extend([
'p_no_intra_sup_a', 'p_no_inter_sup_a', 'p_no_intra_sup_b',
'p_no_inter_sup_b'
])
if inter_support_type == 'realistic':
ml_stats_header.extend(
['p_no_sup_ccs', 'p_no_sup_relays', 'p_no_com_path'])
ml_stats = csv.DictWriter(ml_stats_file,
ml_stats_header,
delimiter='\t',
quoting=csv.QUOTE_MINIMAL)
if write_header is True:
ml_stats.writeheader()
ml_stats_row = {
'sim_group': sim_group,
'instance': instance,
'#nodes_a': node_cnt_a,
'#edges_a': edge_cnt_a,
'#nodes_b': node_cnt_b,
'#edges_b': edge_cnt_b,
'p_atkd': p_atkd,
'p_atkd_a': p_atkd_a,
'p_atkd_b': p_atkd_b,
'p_dead': p_dead,
'p_dead_a': p_dead_a,
'p_dead_b': p_dead_b
}
ml_stats_row.update(all_centr_stats)
if save_attacked_roles is True:
ml_stats_row.update({
'p_atkd_gen': p_atkd_gen,
'p_atkd_ts': p_atkd_ts,
'p_atkd_ds': p_atkd_ds,
'p_atkd_rel': p_atkd_rel,
'p_atkd_cc': p_atkd_cc,
})
ml_stats_row.update({
'lost_deg_a': lost_deg_a,
'lost_deg_b': lost_deg_b,
'lost_indeg_i': lost_indeg_i,
'lost_rel_betw': lost_relay_betw,
'lost_ts_betw': lost_ts_betw
})
if save_death_cause is True:
p_no_intra_sup_a = sf.percent_of_part(intra_sup_deaths_a,
total_dead_a)
p_no_inter_sup_a = sf.percent_of_part(inter_sup_deaths_a,
total_dead_a)
p_no_intra_sup_b = sf.percent_of_part(intra_sup_deaths_b,
total_dead_b)
p_no_inter_sup_b = sf.percent_of_part(inter_sup_deaths_b,
total_dead_b)
ml_stats_row.update({
'p_no_intra_sup_a': p_no_intra_sup_a,
'p_no_inter_sup_a': p_no_inter_sup_a,
'p_no_intra_sup_b': p_no_intra_sup_b,
'p_no_inter_sup_b': p_no_inter_sup_b
})
if inter_support_type == 'realistic':
p_no_sup_ccs = sf.percent_of_part(no_sup_ccs_deaths,
total_dead_a)
p_no_sup_relays = sf.percent_of_part(
no_sup_relays_deaths, total_dead_a)
p_no_com_path = sf.percent_of_part(no_com_path_deaths,
total_dead_a)
ml_stats_row.update({
'p_no_sup_ccs': p_no_sup_ccs,
'p_no_sup_relays': p_no_sup_relays,
'p_no_com_path': p_no_com_path
})
ml_stats.writerow(ml_stats_row)
def calc_indegree_centr_fraction(G, nodes):
tot_indegree = sum(G.in_degree(G.nodes()).values())
nodes_indegree = sum(G.in_degree(nodes).values())
fraction = sf.percent_of_part(nodes_indegree, tot_indegree)
return fraction
def check_split_tolerance(total_item_cnt, chosen_item_cnt, desired_perc, perc_tolerance):
actual_perc = sf.percent_of_part(chosen_item_cnt, total_item_cnt)
perc_deviation = abs(actual_perc - desired_perc)
if perc_deviation > perc_tolerance:
print('Percentage deviation of {}'.format(perc_deviation))
def run(conf_fpath, floader):
global logger
logger.info('conf_fpath = {}'.format(conf_fpath))
conf_fpath = os.path.normpath(conf_fpath)
if os.path.isabs(conf_fpath) is False:
conf_fpath = os.path.abspath(conf_fpath)
if not os.path.isfile(conf_fpath):
raise ValueError(
'Invalid value for parameter "conf_fpath", no such file.\nPath: ' +
conf_fpath)
config = ConfigParser()
config.read(conf_fpath)
global time
time = 0
# read graphml files and instantiate network graphs
seed = config.getint('run_opts', 'seed')
if config.has_option('run_opts', 'save_death_cause'):
save_death_cause = config.getboolean('run_opts', 'save_death_cause')
else:
save_death_cause = False
# TODO: check if directory/files exist
netw_dir = os.path.normpath(config.get('paths', 'netw_dir'))
if os.path.isabs(netw_dir) is False:
netw_dir = os.path.abspath(netw_dir)
netw_a_fname = config.get('paths', 'netw_a_fname')
netw_a_fpath_in = os.path.join(netw_dir, netw_a_fname)
A = floader.fetch_graphml(netw_a_fpath_in, str)
netw_b_fname = config.get('paths', 'netw_b_fname')
netw_b_fpath_in = os.path.join(netw_dir, netw_b_fname)
B = floader.fetch_graphml(netw_b_fpath_in, str)
netw_inter_fname = config.get('paths', 'netw_inter_fname')
netw_inter_fpath_in = os.path.join(netw_dir, netw_inter_fname)
I = floader.fetch_graphml(netw_inter_fpath_in, str)
# if the union graph is needed for this simulation, it's better to have it created in advance and just load it
if config.has_option('paths', 'netw_union_fname'):
netw_union_fname = config.get('paths', 'netw_union_fname')
netw_union_fpath_in = os.path.join(netw_dir, netw_union_fname)
ab_union = floader.fetch_graphml(netw_union_fpath_in, str)
else:
ab_union = None
# read run options
# nodes that must remain alive no matter what
safe_nodes = []
safe_nodes_a = set()
safe_nodes_b = set()
if config.has_section('safe_nodes_opts'):
from_netw = config.get('safe_nodes_opts', 'from_netw')
safe_sel_tactic = config.get('safe_nodes_opts', 'selection_tactic')
safe_nodes = choose_nodes_by_config(config, 'safe_nodes_opts',
from_netw, safe_sel_tactic, A, B,
I, ab_union, floader, netw_dir,
seed)
for node in safe_nodes:
node_netw = I.node[node]['network']
if node_netw == A.graph['name']:
# put the node name in a list, or each char becomes a separate set element
safe_nodes_a.update([node])
elif node_netw == B.graph['name']:
safe_nodes_b.update([node])
if len(safe_nodes) > 0 and config.has_option('run_opts', 'attacks'):
logger.info(
'Nodes marked as safe will not be attacked, so if they get selected to be attacked, the number '
'of attacked nodes will be lower than specified!')
attacked_netw = config.get('run_opts', 'attacked_netw')
attack_tactic = config.get('run_opts', 'attack_tactic')
attacked_nodes = choose_nodes_by_config(config, 'run_opts', attacked_netw,
attack_tactic, A, B, I, ab_union,
floader, netw_dir, seed)
intra_support_type = config.get('run_opts', 'intra_support_type')
if intra_support_type == 'cluster_size':
min_cluster_size = config.getint('run_opts', 'min_cluster_size')
inter_support_type = config.get('run_opts', 'inter_support_type')
# read output paths
results_dir = os.path.normpath(config.get('paths', 'results_dir'))
if os.path.isabs(results_dir) is False:
results_dir = os.path.abspath(results_dir)
sf.ensure_dir_exists(results_dir)
# run_stats is a file used to save step-by-step details about this run
run_stats_fpath = ''
run_stats_file = None
run_stats = None
if config.has_option('paths', 'run_stats_fname'):
run_stats_fname = config.get('paths', 'run_stats_fname')
run_stats_fpath = os.path.join(results_dir, run_stats_fname)
# end_stats is a file used to save a single line (row) of statistics at the end of the simulation
end_stats_fpath = ''
if config.has_option('paths', 'end_stats_fpath'):
end_stats_fpath = os.path.normpath(
config.get('paths', 'end_stats_fpath'))
if os.path.isabs(end_stats_fpath) is False:
end_stats_fpath = os.path.abspath(end_stats_fpath)
sf.ensure_dir_exists(os.path.dirname(end_stats_fpath))
# ml_stats is similar to end_stats as it is used to write a line at the end of the simulation,
# but it gathers statistics used for machine learning
ml_stats_fpath = ''
if config.has_option('paths', 'ml_stats_fpath'):
ml_stats_fpath = os.path.normpath(config.get('paths',
'ml_stats_fpath'))
if os.path.isabs(ml_stats_fpath) is False:
ml_stats_fpath = os.path.abspath(ml_stats_fpath)
# read information used to identify this simulation run
sim_group = config.getint('misc', 'sim_group')
instance = config.getint('misc', 'instance')
batch_conf_fpath = ''
if config.has_option('paths', 'batch_conf_fpath'):
batch_conf_fpath = config.get('paths', 'batch_conf_fpath')
if config.has_option('misc', 'run'):
run_num = config.getint('misc', 'run')
else:
run_num = 0
# statistics meant for machine learning are stored in this dictionary
ml_stats = {
'batch_conf_fpath': batch_conf_fpath,
'sim_group': sim_group,
'instance': instance,
'run': run_num,
'seed': seed
}
# stability check
unstable_nodes = set()
if inter_support_type == 'node_interlink':
unstable_nodes.update(find_nodes_without_inter_links(A, I))
# elif inter_support_type == 'cluster_interlink':
# unstable_nodes.update(find_nodes_in_unsupported_clusters(A, I))
elif inter_support_type == 'realistic':
unstable_nodes.update(find_uncontrolled_pow_nodes(A, B, I))
else:
raise ValueError('Invalid value for parameter "inter_support_type": ' +
inter_support_type)
if intra_support_type == 'giant_component':
unstable_nodes.update(find_nodes_not_in_giant_component(A))
elif intra_support_type == 'cluster_size':
unstable_nodes.update(
find_nodes_in_smaller_clusters(A, min_cluster_size))
elif intra_support_type == 'realistic':
unstable_nodes.update(find_unpowered_substations(A))
else:
raise ValueError('Invalid value for parameter "intra_support_type": ' +
intra_support_type)
if inter_support_type == 'node_interlink':
unstable_nodes.update(find_nodes_without_inter_links(B, I))
# elif inter_support_type == 'cluster_interlink':
# unstable_nodes.update(find_nodes_in_unsupported_clusters(B, I))
elif inter_support_type == 'realistic':
unstable_nodes.update(find_nodes_without_inter_links(B, I))
if intra_support_type == 'giant_component':
unstable_nodes.update(find_nodes_not_in_giant_component(B))
elif intra_support_type == 'cluster_size':
unstable_nodes.update(
find_nodes_in_smaller_clusters(B, min_cluster_size))
# TODO: find out why we skipped / did not implement stability checks for this case
# elif intra_support_type == 'realistic':
# unstable_nodes.update(list())
# remove nodes that can't fail
if len(safe_nodes_a) > 0 or len(safe_nodes_b) > 0:
unstable_nodes -= safe_nodes_a.union(safe_nodes_b)
if len(unstable_nodes) > 0:
logger.debug(
'Time {}) {} nodes unstable before the initial attack: {}'.format(
time, len(unstable_nodes),
sorted(unstable_nodes, key=sf.natural_sort_key)))
total_dead_a = 0
total_dead_b = 0
intra_sup_deaths_a = 0
intra_sup_deaths_b = 0
inter_sup_deaths_a = 0
inter_sup_deaths_b = 0
# only used if save_death_cause is True and inter_support_type == 'realistic'
no_sup_ccs_deaths = 0
no_sup_relays_deaths = 0
no_com_path_deaths = 0
base_node_cnt_a = A.number_of_nodes()
base_node_cnt_b = B.number_of_nodes()
# execute simulation of failure propagation
try:
if run_stats_fpath:
run_stats_file = open(run_stats_fpath, 'wb')
run_stats_header = ['time', 'dead']
run_stats = csv.DictWriter(run_stats_file,
run_stats_header,
delimiter='\t',
quoting=csv.QUOTE_MINIMAL)
run_stats.writeheader()
time += 1
# perform initial attack
attacked_nodes_a = [] # nodes in network A hit by the initial attack
attacked_nodes_b = [] # nodes in network B hit by the initial attack
dead_nodes_a = []
dead_nodes_b = []
for node in attacked_nodes:
node_netw = I.node[node]['network']
if node_netw == A.graph['name']:
attacked_nodes_a.append(node)
elif node_netw == B.graph['name']:
attacked_nodes_b.append(node)
else:
raise RuntimeError(
'Node {} network "{}" marked in inter graph is neither A nor B'
.format(node, node_netw))
if ml_stats_fpath: # if this string is not empty
ml_stats.update({
'#atkd':
len(attacked_nodes_a) + len(attacked_nodes_b),
'#atkd_a':
len(attacked_nodes_a),
'#atkd_b':
len(attacked_nodes_b)
})
ml_stats.update({
'atkd_nodes_a': attacked_nodes_a,
'atkd_nodes_b': attacked_nodes_b
})
ml_stats.update(
calc_atk_centr_stats(A.graph['name'], B.graph['name'],
I.graph['name'], ab_union.graph['name'],
attacked_nodes_a, attacked_nodes_b,
floader, netw_dir))
result_key_by_role = {
'generator': 'p_atkd_gen',
'transmission_substation': 'p_atkd_ts',
'distribution_substation': 'p_atkd_ds'
}
ml_stats.update(
calc_atkd_percent_by_role(A, attacked_nodes_a,
result_key_by_role))
result_key_by_role = {
'relay': 'p_atkd_rel',
'controller': 'p_atkd_cc'
}
ml_stats.update(
calc_atkd_percent_by_role(B, attacked_nodes_b,
result_key_by_role))
total_dead_a = len(attacked_nodes_a)
if total_dead_a > 0:
dead_nodes_a.extend(attacked_nodes_a)
A.remove_nodes_from(attacked_nodes_a)
logger.info(
'Time {}) {} nodes of network {} failed because of initial attack: {}'
.format(time, total_dead_a, A.graph['name'],
sorted(attacked_nodes_a, key=sf.natural_sort_key)))
total_dead_b = len(attacked_nodes_b)
if total_dead_b > 0:
dead_nodes_b.extend(attacked_nodes_b)
B.remove_nodes_from(attacked_nodes_b)
logger.info(
'Time {}) {} nodes of network {} failed because of initial attack: {}'
.format(time, total_dead_b, B.graph['name'],
sorted(attacked_nodes_b, key=sf.natural_sort_key)))
if total_dead_a == total_dead_b == 0:
logger.info('Time {}) No nodes were attacked'.format(time))
I.remove_nodes_from(attacked_nodes)
# save_state(time, A, B, I, results_dir)
if run_stats is not None:
run_stats.writerow({'time': time, 'dead': attacked_nodes})
updated = True
time += 1
# phase checks
while updated is True:
updated = False
# inter checks for network A
if inter_support_type == 'node_interlink':
unsupported_nodes_a = find_nodes_without_inter_links(A, I)
# elif inter_support_type == 'cluster_interlink':
# unsupported_nodes_a = find_nodes_in_unsupported_clusters(A, I)
elif inter_support_type == 'realistic':
unsupported_nodes_a = find_uncontrolled_pow_nodes(
A, B, I, save_death_cause)
if save_death_cause is False:
unsupported_nodes_a = remove_list_items(
unsupported_nodes_a, safe_nodes_a)
elif inter_support_type == 'realistic': # save_death_cause is True
remove_items_from_lists_in_dict(unsupported_nodes_a,
safe_nodes_a)
no_sup_ccs_deaths += len(unsupported_nodes_a['no_sup_ccs'])
no_sup_relays_deaths += len(
unsupported_nodes_a['no_sup_relays'])
no_com_path_deaths += len(unsupported_nodes_a['no_com_path'])
temp_list = []
for node_list in unsupported_nodes_a.values(
): # convert dictionary of lists into a simple list
temp_list.extend(node_list)
unsupported_nodes_a = temp_list
failed_cnt_a = len(unsupported_nodes_a)
if failed_cnt_a > 0:
logger.info(
'Time {}) {} nodes of network {} failed for lack of inter support: {}'
.format(
time, failed_cnt_a, A.graph['name'],
sorted(unsupported_nodes_a, key=sf.natural_sort_key)))
total_dead_a += failed_cnt_a
inter_sup_deaths_a += failed_cnt_a
dead_nodes_a.extend(unsupported_nodes_a)
A.remove_nodes_from(unsupported_nodes_a)
I.remove_nodes_from(unsupported_nodes_a)
updated = True
# save_state(time, A, B, I, results_dir)
if run_stats is not None:
run_stats.writerow({
'time': time,
'dead': unsupported_nodes_a
})
time += 1
# intra checks for network A
if intra_support_type == 'giant_component':
unsupported_nodes_a = find_nodes_not_in_giant_component(A)
elif intra_support_type == 'cluster_size':
unsupported_nodes_a = find_nodes_in_smaller_clusters(
A, min_cluster_size)
elif intra_support_type == 'realistic':
unsupported_nodes_a = find_unpowered_substations(A)
unsupported_nodes_a = remove_list_items(unsupported_nodes_a,
safe_nodes_a)
failed_cnt_a = len(unsupported_nodes_a)
if failed_cnt_a > 0:
logger.info(
'Time {}) {} nodes of network {} failed for lack of intra support: {}'
.format(
time, failed_cnt_a, A.graph['name'],
sorted(unsupported_nodes_a, key=sf.natural_sort_key)))
total_dead_a += failed_cnt_a
intra_sup_deaths_a += failed_cnt_a
dead_nodes_a.extend(unsupported_nodes_a)
A.remove_nodes_from(unsupported_nodes_a)
I.remove_nodes_from(unsupported_nodes_a)
updated = True
# save_state(time, A, B, I, results_dir)
if run_stats is not None:
run_stats.writerow({
'time': time,
'dead': unsupported_nodes_a
})
time += 1
# inter checks for network B
if inter_support_type == 'node_interlink':
unsupported_nodes_b = find_nodes_without_inter_links(B, I)
# elif inter_support_type == 'cluster_interlink':
# unsupported_nodes_b = find_nodes_in_unsupported_clusters(B, I)
elif inter_support_type == 'realistic':
unsupported_nodes_b = find_nodes_without_inter_links(B, I)
unsupported_nodes_b = remove_list_items(unsupported_nodes_b,
safe_nodes_b)
failed_cnt_b = len(unsupported_nodes_b)
if failed_cnt_b > 0:
logger.info(
'Time {}) {} nodes of network {} failed for lack of inter support: {}'
.format(
time, failed_cnt_b, B.graph['name'],
sorted(unsupported_nodes_b, key=sf.natural_sort_key)))
total_dead_b += failed_cnt_b
inter_sup_deaths_b += failed_cnt_b
dead_nodes_b.extend(unsupported_nodes_b)
B.remove_nodes_from(unsupported_nodes_b)
I.remove_nodes_from(unsupported_nodes_b)
updated = True
# save_state(time, A, B, I, results_dir)
if run_stats is not None:
run_stats.writerow({
'time': time,
'dead': unsupported_nodes_b
})
time += 1
# intra checks for network B
if intra_support_type == 'giant_component':
unsupported_nodes_b = find_nodes_not_in_giant_component(B)
elif intra_support_type == 'cluster_size':
unsupported_nodes_b = find_nodes_in_smaller_clusters(
B, min_cluster_size)
elif intra_support_type == 'realistic':
# in the realistic model, telecom nodes can survive without intra support, they just cant't communicate
unsupported_nodes_b = []
unsupported_nodes_b = remove_list_items(unsupported_nodes_b,
safe_nodes_b)
failed_cnt_b = len(unsupported_nodes_b)
if failed_cnt_b > 0:
logger.info(
'Time {}) {} nodes of network {} failed for lack of intra support: {}'
.format(
time, failed_cnt_b, B.graph['name'],
sorted(unsupported_nodes_b, key=sf.natural_sort_key)))
total_dead_b += failed_cnt_b
intra_sup_deaths_b += failed_cnt_b
dead_nodes_b.extend(unsupported_nodes_b)
B.remove_nodes_from(unsupported_nodes_b)
I.remove_nodes_from(unsupported_nodes_b)
updated = True
# save_state(time, A, B, I, results_dir)
if run_stats is not None:
run_stats.writerow({
'time': time,
'dead': unsupported_nodes_b
})
time += 1
except:
if run_stats_file is not None:
run_stats_file.close()
raise
# save_state('final', A, B, I, results_dir)
# write statistics about the final result
if end_stats_fpath: # if this string is not empty
end_stats_file_existed = os.path.isfile(end_stats_fpath)
with open(end_stats_fpath, 'ab') as end_stats_file:
end_stats_header = [
'batch_conf_fpath', 'sim_group', 'instance', 'run', '#dead',
'#dead_a', '#dead_b'
]
if save_death_cause is True:
end_stats_header.extend([
'no_intra_sup_a', 'no_inter_sup_a', 'no_intra_sup_b',
'no_inter_sup_b'
])
if inter_support_type == 'realistic':
end_stats_header.extend(
['no_sup_ccs', 'no_sup_relays', 'no_com_path'])
end_stats_writer = csv.DictWriter(end_stats_file,
end_stats_header,
delimiter='\t',
quoting=csv.QUOTE_MINIMAL)
if end_stats_file_existed is False:
end_stats_writer.writeheader()
end_stats_row = {
'batch_conf_fpath': batch_conf_fpath,
'sim_group': sim_group,
'instance': instance,
'run': run_num,
'#dead': total_dead_a + total_dead_b,
'#dead_a': total_dead_a,
'#dead_b': total_dead_b
}
if save_death_cause is True:
end_stats_row.update({
'no_intra_sup_a': intra_sup_deaths_a,
'no_inter_sup_a': inter_sup_deaths_a,
'no_intra_sup_b': intra_sup_deaths_b,
'no_inter_sup_b': inter_sup_deaths_b
})
if inter_support_type == 'realistic':
end_stats_row.update({
'no_sup_ccs': no_sup_ccs_deaths,
'no_sup_relays': no_sup_relays_deaths,
'no_com_path': no_com_path_deaths
})
end_stats_writer.writerow(end_stats_row)
if ml_stats_fpath: # if this string is not empty
# percentages of dead nodes over the initial number of nodes in the graph
ml_stats['p_dead_a'] = sf.percent_of_part(total_dead_a,
base_node_cnt_a)
ml_stats['p_dead_b'] = sf.percent_of_part(total_dead_b,
base_node_cnt_b)
ml_stats['p_dead'] = sf.percent_of_part(
total_dead_a + total_dead_b, base_node_cnt_a + base_node_cnt_b)
ml_stats['dead_nodes_a'] = dead_nodes_a
ml_stats['dead_nodes_b'] = dead_nodes_b
ml_stats['dead_count_a'] = len(dead_nodes_a)
ml_stats['dead_count_b'] = len(dead_nodes_b)
ml_stats['dead_count'] = len(dead_nodes_a) + len(dead_nodes_b)
if config.has_section('safe_nodes_opts'):
ml_stats['safe_count'] = len(safe_nodes)
ml_stats_file_existed = os.path.isfile(ml_stats_fpath)
with open(ml_stats_fpath, 'ab') as ml_stats_file:
# sort statistics columns by name so they can be more found easily in the output file
ml_stats_header = sorted(ml_stats.keys(), key=sf.natural_sort_key)
ml_stats_writer = csv.DictWriter(ml_stats_file,
ml_stats_header,
delimiter='\t',
quoting=csv.QUOTE_MINIMAL)
if ml_stats_file_existed is False:
ml_stats_writer.writeheader()
ml_stats_writer.writerow(ml_stats)
def calc_atk_centr_stats(name_A, name_B, name_I, name_AB, attacked_nodes_a,
attacked_nodes_b, floader, netw_dir):
attacked_nodes = attacked_nodes_a + attacked_nodes_b
centr_stats = {}
# load files with precalculated centrality metrics
centr_fpath_a = os.path.join(netw_dir,
'node_centrality_{}.json'.format(name_A))
centr_info_a = floader.fetch_json(centr_fpath_a)
centr_fpath_b = os.path.join(netw_dir,
'node_centrality_{}.json'.format(name_B))
centr_info_b = floader.fetch_json(centr_fpath_b)
centr_fpath_ab = os.path.join(netw_dir,
'node_centrality_{}.json'.format(name_AB))
centr_info_ab = floader.fetch_json(centr_fpath_ab)
centr_fpath_i = os.path.join(netw_dir,
'node_centrality_{}.json'.format(name_I))
centr_info_i = floader.fetch_json(centr_fpath_i)
centr_fpath_gen = os.path.join(netw_dir, 'node_centrality_misc.json')
centr_info_misc = floader.fetch_json(centr_fpath_gen)
node_cnt_a = centr_info_a['node_count']
centr_stats['p_atkd_a'] = sf.percent_of_part(len(attacked_nodes_a),
node_cnt_a)
node_cnt_b = centr_info_b['node_count']
centr_stats['p_atkd_b'] = sf.percent_of_part(len(attacked_nodes_b),
node_cnt_b)
centr_name = 'betweenness_centrality'
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes_a, centr_name,
'atkd_betw_c_a', centr_info_a))
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes_b, centr_name,
'atkd_betw_c_b', centr_info_b))
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes, centr_name, 'atkd_betw_c_ab',
centr_info_ab))
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes, centr_name, 'atkd_betw_c_i',
centr_info_i))
centr_name = 'closeness_centrality'
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes_a, centr_name,
'atkd_clos_c_a', centr_info_a))
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes_b, centr_name,
'atkd_clos_c_b', centr_info_b))
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes, centr_name, 'atkd_clos_c_ab',
centr_info_ab))
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes, centr_name, 'atkd_clos_c_i',
centr_info_i))
centr_name = 'degree_centrality'
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes_a, centr_name, 'atkd_deg_c_a',
centr_info_a))
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes_b, centr_name, 'atkd_deg_c_b',
centr_info_b))
centr_name = 'indegree_centrality'
if centr_name in centr_info_ab:
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes, centr_name,
'atkd_indeg_c_ab', centr_info_ab))
if centr_name in centr_info_i:
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes, centr_name,
'atkd_indeg_c_i', centr_info_i))
centr_name = 'katz_centrality'
if centr_name in centr_info_ab:
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes, centr_name,
'atkd_katz_c_ab', centr_info_ab))
if centr_name in centr_info_i:
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes, centr_name,
'atkd_katz_c_i', centr_info_i))
centr_name = 'relay_betweenness_centrality'
if centr_name in centr_info_misc:
centr_info_misc['node_count'] = centr_info_misc[
'relay_count'] # trick to reuse the same logic
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes, centr_name,
'atkd_rel_betw_c', centr_info_misc))
centr_name = 'transm_subst_betweenness_centrality'
if centr_name in centr_info_misc:
centr_info_misc['node_count'] = centr_info_misc[
'transmission_substation_count'] # trick
centr_stats.update(
calc_atk_centrality_stats(attacked_nodes, centr_name,
'atkd_ts_betw_c', centr_info_misc))
return centr_stats
def calc_atk_centrality_stats(attacked_nodes, centrality_name,
result_key_suffix, centrality_info):
global logger
centr_stats = {} # dictionary used to index statistics with shorter names
node_cnt = centrality_info['node_count']
centr_by_node = centrality_info[centrality_name]
# sum the centrality scores of the attacked nodes
atkd_centr_sum = 0.0
total_centr = centrality_info['total_' + centrality_name]
for node in attacked_nodes:
atkd_centr_sum += centr_by_node[node]
stat_name = 'sum_' + result_key_suffix
centr_stats[stat_name] = atkd_centr_sum
stat_name = 'p_tot_' + result_key_suffix
centr_stats[stat_name] = sf.percent_of_part(atkd_centr_sum, total_centr)
# count how many nodes were attacked in each quintile
quintiles = centrality_info[centrality_name + '_quintiles']
atkd_cnt_by_quintile = [0] * 5
quintiles_set = set(quintiles)
if len(quintiles_set) == len(quintiles):
for node in attacked_nodes:
node_quintile = 0
node_centr = centr_by_node[node]
for quintile_val in quintiles:
if node_centr > quintile_val:
node_quintile += 1
else:
break
atkd_cnt_by_quintile[node_quintile] += 1
else:
# this is a corner case presented by graphs in which many nodes have identical centrality scores, like in
# random regular graphs. To determine what quintile each node belongs to, we check their position in our
# deterministic ranking, where they are sorted by their centrality score first and then by their id
# logger.info('There are only {} different quintiles for centrality {}'.format(len(quintiles_set),
# centrality_name))
ranked_nodes = centrality_info[centrality_name + '_rank']
# find the positions that separate the quintiles
rank_of_quintiles = percentile(range(0, node_cnt),
[20, 40, 60, 80]).tolist()
logger.debug('rank_of_quintiles {}'.format(rank_of_quintiles))
for node in attacked_nodes:
node_quintile = 0
node_rank = ranked_nodes.index(node)
for quintile_rank in rank_of_quintiles:
if node_rank > quintile_rank:
node_quintile += 1
else:
break
atkd_cnt_by_quintile[node_quintile] += 1
logger.debug('atkd_cnt_by_quintile {}'.format(atkd_cnt_by_quintile))
nodes_in_quintile = 1.0 * node_cnt / 5
logger.debug('nodes_in_quintile {}'.format(nodes_in_quintile))
# calculate the percentage of nodes attacked in each quintile
atkd_perc_of_quintile = [0.0] * 5
for i in range(0, 5):
atkd_perc_of_quintile[i] = sf.percent_of_part(atkd_cnt_by_quintile[i],
nodes_in_quintile)
logger.debug('atkd_perc_of_quintile {}'.format(atkd_perc_of_quintile))
# p_q_1 is fraction of nodes attacked in the first quintile
for i, val in enumerate(atkd_perc_of_quintile):
stat_name = 'p_q_{}_{}'.format(i + 1, result_key_suffix)
centr_stats[stat_name] = val
return centr_stats