def test_dot_graph_load(self):
# get the graph
data_dir = os.path.join(self.data_dir, "audiology.txt")
df = pd.read_table(data_dir, sep="\t")
dot_str = self.pc_util.algo_fges_discrete(df)
graph = nx_agraph.from_agraph(pygraphviz.AGraph(dot_str))
self.assertTrue(graph is not None, "Nx did not load graph data.")
def get_networkx_graph_from_pygraphviz_graph():
'''It's annoying that NetworkX Graph objects only have a string representation based on the <Graph>.graph['name'] attribute'''
g = nx.Graph([(1, 2), (7, 8), (2, 8)])
g.graph['name'] = 'bar'
pygraphviz_graph = nx_agraph.to_agraph(g)
networkx_graph = nx_agraph.from_agraph(pygraphviz_graph)
print(type(networkx_graph)) # <class 'networkx.classes.graph.Graph'>
print(networkx_graph) # bar
print(networkx_graph.edges()) # [('1', '2'), ('2', '8'), ('7', '8')]
def regex2dfa(reg_ex, letter='q'):
transfer_file = tempfile.NamedTemporaryFile(mode='w+')
command = 'java -jar {}/regex2dfa.jar "{}" {}'.format(
JAR_DIR, reg_ex, transfer_file.name)
os.system(command)
with open(transfer_file.name) as fname:
dot = fname.read()
print(dot, file=open('{}.dot'.format(transfer_file.name), 'w'))
return nx_agraph.from_agraph(pygraphviz.AGraph(dot))
def read_graph(dot_string, is_directed, is_weighted):
'''
Reads the input network in networkx.
'''
G = nx_agraph.from_agraph(pygraphviz.AGraph(dot_string))
for source, target in G.edges():
for key in G[source][target].keys():
G[source][target][key]['weight'] = 1
if not is_directed:
G = G.to_undirected()
return G
def to_dfa(self, regex):
response = requests.get('{}/{}/{}/{}/'.format(self._URL, self._service,
self._letter, regex))
dot = response.json()['dfaInDot']
dfa = nx_agraph.from_agraph(pygraphviz.AGraph(dot))
accept_states = [
n for n in dfa.nodes()
if dfa.nodes.data('shape')[n] == 'doublecircle'
]
dfa_triple = collections.namedtuple('DFA', ['dot', 'start', 'accepts'])
return dfa_triple(dfa, self._initial, accept_states)
def __init__(self, regex, letter='q'):
self.regex = regex
self._f = tempfile.NamedTemporaryFile(mode='w+')
command = 'java -jar src/java-lib/regex2dfa.jar "{}" {}'.format(
regex, self._f.name)
os.system(command)
with open(self._f.name) as fname:
dot = fname.read()
print(dot, file=open('{}.dot'.format(self._f.name), 'w'))
self._dfa = nx_agraph.from_agraph(pygraphviz.AGraph(dot))
self._accept_states = [
n for n in self._dfa.nodes()
if self._dfa.nodes.data('shape')[n] == 'doublecircle'
]
self._states = [n for n in self._dfa.nodes()]
def __call__(self, graph_data: str,
number_input_symbols: int = None,
graph_data_format: str = 'dot_string') -> FDFA:
"""
Returns an initialized FDFA instance given the graph_data
graph_data and graph_data_format must match
:param graph_data: The string containing graph data.
Could be a filename or just the raw
data
:param number_input_symbols: The number of input symbols to the
FDFA needed to compute the correct
frequency flows in the case of
cycles.
Only really optional when using a
graph_data_format
that already has this information.
:param graph_data_format: The graph data file format.
{'dot_file', 'dot_string',
'learning_interface'}
:returns: instance of an initialized FDFA object
:raises ValueError: checks if graph_data and
graph_data_format have a compatible
data loader.
:raises ValueError: checks, based on graph_data_format,
whether it is legal to not specify
the number_input_symbols.
"""
has_number_input_symbols = number_input_symbols is not None
if graph_data_format == 'dot_string':
graph = nx_agraph.from_agraph(pygraphviz.AGraph(string=graph_data))
elif graph_data_format == 'dot_file':
graph = read_dot(graph_data)
elif graph_data_format == 'learning_interface':
learning_interface = graph_data
graph = read_dot(learning_interface.learned_model_filepath)
number_input_symbols = learning_interface.num_training_examples
has_number_input_symbols = True
else:
msg = 'graph_data_format ({}) must be one of: "dot_file", ' + \
'"dot_string"'.format(graph_data_format)
raise ValueError(msg)
if not has_number_input_symbols:
msg = f'must provide the number_input_symbols to load a FDFA'
raise ValueError(msg)
# these are not things that are a part of flexfringe's automaton
# data model, so give them default values
final_transition_sym = DEFAULT_FINAL_TRANS_SYMBOL
empty_transition_sym = DEFAULT_EMPTY_TRANS_SYMBOL
config_data = FDFA.load_flexfringe_data(graph,
number_input_symbols,
final_transition_sym,
empty_transition_sym)
config_data['final_transition_sym'] = final_transition_sym
config_data['empty_transition_sym'] = empty_transition_sym
nodes_have_changed = (self.nodes != config_data['nodes'])
edges_have_changed = (self.edges != config_data['edges'])
no_instance_loaded_yet = (self._instance is None)
if no_instance_loaded_yet or nodes_have_changed or edges_have_changed:
# saving these so we can just return initialized instances if the
# underlying data has not changed
self.nodes = config_data['nodes']
self.edges = config_data['edges']
self._instance = FDFA(**config_data)
return self._instance
dotFileExtensions = ['.dot', '.gv']
dataFileName = 'data.json'
if len(sys.argv) == 3:
dotFolderPath = os.path.abspath(sys.argv[1])
jsonFolderPath = os.path.abspath(sys.argv[2])
if not os.path.exists(jsonFolderPath):
os.mkdir(jsonFolderPath)
dotFiles = filesWithExtensions(dotFolderPath, dotFileExtensions)
counter = 0
for dotFile in dotFiles:
dotFilePath = dotFolderPath + '/' + dotFile
try:
dot_graph = pgv.AGraph(dotFilePath)
graph_netx = from_agraph(dot_graph)
except (ValueError, DotError) as e:
try:
graph_netx = read_dot(dotFilePath)
except (ValueError, DotError) as f:
print(dotFile + ' not in graphviz format')
continue
graph_json = json_graph.node_link_data(graph_netx) #dot_graph)
filename = dotFile[:dotFile.rfind('.')]
json.dump(graph_json,
open(jsonFolderPath + '/' + filename + '.json', 'w'),
indent=2)
print(filename + '.json converted')
counter += 1
with open(dataFileName, 'w') as jsonFile:
def main():
parser = argparse.ArgumentParser(
description=
'Simulate run of routing strategies\n\n python RL.py <EPISODES> <GAMMA> <EPOCH> <BATCHSIZE> <BUFFER> <TOPOLOGY> <TSIM> <ROUTING> \n \n === topologies options === \n [3cycle] \n [4cycle] \n [4diag] \n [6s4hMultiSwitch] \n [Aarnet] \n [Abilene] \n \n === routing options ===\n[-rl] \t\t run reinforcement learning \n[-so] \t\t run semi-oblivious \n[-ecmp] \t run ecmp \n[-ospf] \t run ospf',
formatter_class=RawTextHelpFormatter)
parser.add_argument(
'episodes',
type=int,
help='Number of episodes to train the Reinforcement Learning algorith')
parser.add_argument('gamma', type=float, help='Discount factor')
parser.add_argument('epochs', type=int, help='Number of epochs')
parser.add_argument('batchSize',
type=int,
help='batch size of the Experience Replay buffer')
parser.add_argument(
'buffer',
type=int,
help='size of the replay buffer (for the Experience Replay)')
parser.add_argument('topology', type=str, help='Simulation topology')
parser.add_argument('simTime', type=int, help='Simulation Time')
parser.add_argument('routing', type=str, help='Routing protocol')
args = parser.parse_args()
if args.gamma > 1 or args.gamma < 0:
parser.error("gamma cannot be larger than 1 and smaller than 0")
parameters = [
args.episodes, args.gamma, args.epochs, args.batchSize, args.buffer,
args.routing, args.simTime
]
print('parameters', parameters)
initialnFlow = 3 #10 #maximum number of flows that can coexist in the system (for the training)
# RL1 = ReinforcementLearning(initialnFlow, 0)
# model, nNode, points_list, bandwidth_list, delay_list = RL1.training(parameters)
dotFormat = './topologies/' + args.topology + '.dot'
G = nx_agraph.from_agraph(pygraphviz.AGraph(dotFormat))
# pos = nx.spring_layout(G)
# nx.draw_networkx_nodes(G,pos)
# nx.draw_networkx_edges(G,pos)
# nx.draw_networkx_labels(G,pos)
labels = nx.get_edge_attributes(G, 'weight')
#topology parameters
node_name = nx.get_node_attributes(G, 'type')
print('lables', node_name)
nNode = nx.number_of_nodes(G)
print('nnode', nNode)
nHost = 0
print('nodi: ', nNode)
for attr in node_name:
# print('attr', attr)
if (attr[0] == 'h'):
nHost += 1
nSwitch = nNode - nHost
# print('nhost, nswitch', nHost, nSwitch)
#newgraph with nodes' name from 0 to nNode-1
myG = G
mapping = {}
cuonter_h = 1
cuonter_s = 1
num_h = 0
num_s = num_h + nHost
index = 0
while index < nNode:
for n in G.nodes():
if (n[0] == 's' and n[1:] == str(cuonter_s)):
mapping[n] = num_s
cuonter_s += 1
num_s += 1
index += 1
elif (n[0] == 'h' and n[1:] == str(cuonter_h)):
mapping[n] = num_h
cuonter_h += 1
num_h += 1
index += 1
myG = nx.relabel_nodes(G, mapping)
model = None
RL1 = None
if args.routing == 'rl':
#trainig RL!!
activeFlows = initialnFlow - 1
RL1 = ReinforcementLearning(initialnFlow, myG, 0)
model = RL1.training(parameters, nNode, myG, nHost, activeFlows)
# nx.draw_networkx_edge_labels(G,pos,edge_labels=labels)
# plt.show()
#save name
#save_path = 'C:/example/'
#name_of_file = raw_input("What is the name of the file: ")
#completeName = os.path.join(save_path, name_of_file+".txt")
#file1 = open(completeName, "w")
# toFile = raw_input(model)
# file1.write(toFile)
GM = GraphManager(nNode)
#start generation packet and normal simulation
packetsGen = packetsGeneration(initialnFlow)
failedRL = packetsGen.generation(GM, nNode, nHost, nSwitch, myG, model,
parameters, RL1, dotFormat)
def output_graphs(callers, r2):
# first, generate a supergraph of all nodes for each caller
for func, func_caller in callers.items():
logging.info("Func: 0x{:04x} Caller: {}".format(func, func_caller))
for addr, callee in func_caller.callees.items():
logging.info("Addr: 0x{:04x} Callee: {}".format(addr, callee))
for func, func_caller in callers.items():
func_str = '0x{:04x}'.format(func)
logging.info("Seeking to address {} in radare.".format(func_str))
r2.cmd("s {}".format(func_str))
logging.debug("Current addr: {}".format(
r2.cmd("s"))) # seek to the address of this func
logging.info("Creating main caller JSON, Disassembly")
r2.cmd('af-') # clean analysis data
r2.cmd('aa')
#r2.cmd('af')
#r2.cmd('sp')
func_caller.json = r2.cmd('agdj') # pull JSON disassembly from R2
func_caller.dot = r2.cmd('agd') # pull dot for function from R2
func_caller.graph = nx_agraph.from_agraph(
pygraphviz.AGraph(func_caller.dot)) # pull graph into networkx
new_path = '{}-{}'.format(func_str, func_caller.count)
if not os.path.exists(new_path):
os.makedirs(new_path)
if not os.curdir == new_path:
os.chdir(new_path)
proc_string = "gvpack -o {}/{}_master.dot {}/{}.dot".format(
new_path, func_str, new_path, func_str)
#logging.debug("Path object for CALLER: {}".format(new_path))
f1 = open("{}.json".format(func_str), "w")
f2 = open("{}.dot".format(func_str), "w")
f1.write(func_caller.json)
f2.write(func_caller.dot)
f1.close()
f2.close()
for addr, callee in func_caller.callees.items():
try:
addr_str = str('0x{:04x}'.format(callee.dest_addr))
except ValueError:
addr_str = str('0x{}'.format(callee.dest_addr))
r2.cmd("s {}".format(addr_str))
logging.debug("Current addr: {}".format(
r2.cmd("s"))) # seek to the address of this func
r2.cmd('af-') # clean analysis data
r2.cmd('aa')
#r2.cmd('af')
#r2.cmd('sp') # seek to func identified here
callee.json = r2.cmd('agdj')
callee.dot = r2.cmd('agd')
sub_path = '{}'.format(addr_str)
callee.graph = nx_agraph.from_agraph(pygraphviz.AGraph(
callee.dot)) # pull graph into networkx
if not os.path.exists(sub_path):
os.makedirs(sub_path)
os.chdir(sub_path)
proc_string = proc_string + (" {}/{}/{}.dot".format(
new_path, '0x{:04x}'.format(addr), sub_path))
f3 = open("{}.json".format(sub_path), "w")
f4 = open("{}.dot".format(sub_path), "w")
check_call([
'dot', '-Tpng', '-o', "{}.png".format(sub_path),
"{}.dot".format(sub_path)
])
f3.write(callee.json)
f4.write(callee.dot)
#callee.graph = nx_agraph.read_dot(f4)
#caller.master = nx.compose(func_caller.graph, callee.graph)
f3.close()
f4.close()
os.chdir("..")
#print proc_string
#process = subprocess.Popen(proc_string.split(), stdout=subprocess.PIPE)
#output, error = process.communicate()
#logging.info(output)
#logging.debug(error)
os.chdir("..")
# print func_caller.dot
# print func_caller.graph.edges()
# print func_caller.master.edges()
cwd = os.getcwd()
os.chdir(cwd)
return callers
def create_graph(binary):
"""
Adds syscall nodes into the graph generated from the passed in binary and then
links the syscall nodes to the according nodes that call the syscall
Args:
binary (string): file path for the binary
Returns:
MultiDiGraph with node format as follows -
nodename is a string
node values include 'label' (assembly instructions) and 'shape' = 'record'
"""
og_name = binary
with tempfile.TemporaryDirectory() as dirname:
# get a new binary with dead code removed
binary = strip_dead_code(binary, dirname)
# run Radare2 on the binary to get dotfile contents
radare_pipe = r2pipe.open(binary)
radare_pipe.cmd('aaaaa')
radare_pipe.cmd('s main')
dotContents = radare_pipe.cmd('agfd')
dot_path = Path(dirname) / f'{binary}.dot'
with (dot_path).open(mode='w+') as f:
f.write(dotContents)
png_path = f'~/graphs/{Path(og_name).name}.png'
subprocess.run(f'dot -Tpng {dot_path} -o {png_path}',
shell=True,
check=True)
graph = from_agraph(pygraphviz.AGraph(dotContents))
# add syscall nodes
#for syscall_name in caught_syscalls:
# node_name = 'syscall_{}'.format(syscall_name)
# graph.add_node(
# node_name,
# label='syscall {}'.format(syscall_name),
# shape='record'
# )
# link syscall nodes to original nodes
#for node_name, node_info in graph.nodes.items():
# # skip over our added syscall nodes
# if 'syscall_' in node_name:
# continue
# instructions = node_info.get('label')
# for syscall_name in caught_syscalls:
# instr_regex = r"call\s.*sym.imp.{}".format(syscall_name)
# if re.search(instr_regex, instructions):
# # connect the syscall node to this node
# syscall_node_name = 'syscall_{}'.format(syscall_name)
# graph.add_edge(
# syscall_node_name,
# node_name,
# weight=10
# )
return graph
def get_json(request, task_id): if request.method == 'GET': task = Task.objects.get(id = task_id) dot_graph = nx_agraph.from_agraph(pygraphviz.AGraph(task.dot_rep)) graph_json = json.dumps(json_graph.node_link_data(dot_graph)) return HttpResponse(graph_json, content_type='application/json')