def preprocessing(self):
ego_id = 1684
count = -1
n_clusters = 6
graph = NodeLinkGraph("./data/facebook")
nodes, edges = graph.loadEdges(ego_id, count)
labels = graph.loadCircles(ego_id)
c = Cluster(nodes, edges)
# clusters1 = c.spectral(n_clusters)
clusters1 = c.hierarchical(n_clusters)
# clusters2 = c.agglomerate(nodes, edges, clusters1)
# clusters3 = c.constraint(nodes, edges, labels)
# st2 = time.time()
# print(st2 - st)
nodes = [
Node(nodes[i], int(clusters1[i]), int(labels[i]))
for i in range(len(nodes))
]
nodes.sort(key=lambda n: n.cluster)
edges = [Edge(e[0], e[1], 1.0) for e in edges]
return nodes, edges
def _dummy():
n_points = 240000
n_clusters = 12
seg = int(floor(n_points / n_clusters))
nodes = [Node(i, int(floor(i / seg))) for i in range(n_points)]
edges = []
return nodes, edges
def deeper_conv2d(self,
model,
layer_name,
config,
with_pooling,
kernel_size=3,
filters='same'):
# construct graph
new_graph = model.graph.copy()
if with_pooling == False:
type = 'Conv2D'
else:
type = 'Conv2D_Pooling'
new_node = Node(type=type,
name='new',
config={
'kernel_size': kernel_size,
'filters': filters
})
new_graph.deeper(layer_name, new_node)
# logger.debug(new_graph.to_json())
# construct model
new_model = MyModel(config=config, graph=new_graph)
# inherit weight
# in fact there is no need to get w_conv1 and b_conv1
# what we actually need is only w_conv1's shape
# more specifically, only kh, kw and filters are needed, num_channel is not necessary
node_type = model.graph.get_nodes(layer_name)[0].type
if node_type == 'Conv2D' or node_type == 'Conv2D_Pooling':
w_conv1, b_conv1 = new_model.get_layers(
layer_name)[0].get_weights()
# tensorflow kernel format: [filter_height, filter_width, in_channels, out_channels] channels_last
# theano kernel format: [output_channels, input_channels, filter_rows, filter_columns] channels_first
if K.image_data_format() == "channels_first": # theano format
# convert_kernel function Converts a Numpy kernel matrix from Theano format to TensorFlow format, vise versa
w_conv1 = convert_kernel(w_conv1)
kh, kw, num_channel, filters = w_conv1.shape
elif node_type == 'Group':
kh, kw, filters = 3, 3, model.graph.get_nodes(
layer_name)[0].config['filters']
w_conv2, b_conv2 = new_model.get_layers(
layer_name, next_layer=True)[0].get_weights()
new_w_conv2, new_b_conv2 = Net2Net._deeper_conv2d_weight(
kh=kh, kw=kw, filters=filters)
new_model.get_layers(layer_name, next_layer=True)[0].set_weights(
[new_w_conv2, new_b_conv2])
self.copy_weight(model, new_model)
return new_model
def create_vpc_node(json, region_node):
""" Builder for a vpc node
:param json: json from AWS API
:param region_node: region father node
:return: The vpc node
"""
resource_type = 'Vpc'
name = get_name_from_tags(json.get('Tags'))
# Using json fields to build graphviz label
label = ('"' + resource_type +
'\n' + name +
'\n' + json.get('VpcId') + '"')
return Node(json=json, resource_type=resource_type,
id_type='VpcId', color='orange', label=label,
father=region_node, service='network')
def create_role_node(json, iam):
""" Builder for a role node
:param json: json from AWS API
:param iam: iam father node
:return: the created role node
"""
resource_type = 'Role'
label = '"' + resource_type + ' : ' + json.get('RoleName') + '"'
return Node(json=json,
resource_type=resource_type,
id_type='Arn',
color='plum',
label=label,
father=iam,
service='iam')
def create_group_node(json, iam):
""" Builder for a group node
:param json: json from AWS API
:param iam: iam father node
:return: the created group node
"""
resource_type = 'Group'
label = '"' + resource_type + ' : ' + json.get('GroupName') + '"'
return Node(json=json,
resource_type=resource_type,
id_type='Arn',
color='tomato',
label=label,
father=iam,
service='iam')
def create_user_node(json, iam):
""" Builder for a user node
:param json: json from AWS API
:param iam: iam father node
:return: the created user node
"""
resource_type = 'User'
label = '"' + resource_type + ' : ' + json.get('UserName') + '"'
return Node(json=json,
resource_type=resource_type,
id_type='Arn',
color='turquoise',
label=label,
father=iam,
service='iam')
def create_policy_node(json, iam):
""" Builder for a policy node
:param json: json from AWS API
:param iam: iam father node
:return: the created user node
"""
resource_type = 'Policy'
label = '"' + resource_type + ' : ' + json.get('PolicyName') + '"'
json['CustomId'] = json.get('Arn') + iam.father.id
return Node(json=json,
resource_type=resource_type,
id_type='CustomId',
color='limegreen',
label=label,
father=iam,
service='iam')
def create_subnet_node(json, vpc):
""" Builder for a subnet node
:param json: json from AWS API
:param vpc: vpc father node
:return: The subnet node
"""
resource_type = 'Subnet'
name = get_name_from_tags(json.get('Tags'))
# Using json fields to build graphviz label
label = ('"' + resource_type
+ '\n' + name
+ '\n' + json.get('SubnetId')
+ '\nAZ :' + json.get('AvailabilityZone') + '"')
return Node(json=json, resource_type=resource_type,
id_type='SubnetId', color='lightblue',
label=label, father=vpc, service='network')
def create_access_key_node(json, user):
""" Builder for an access key node
:param json: json from AWS API
:param user: user father node
:return: the created access key node
"""
resource_type = 'AccessKey'
label = ('"Access Key\nUser : '******'UserName') + '\nStatus : ' +
json.get('Status') + '"')
return Node(json=json,
resource_type=resource_type,
id_type=resource_type + 'Id',
color='turquoise',
label=label,
father=user,
service='iam')
def create_mfa_device_node(json, user):
""" Builder for a mfa device node
:param json: json from AWS API
:param user: user father node
:return: the created mfa device node
"""
resource_type = 'MFADevice'
enable_date = json.get('EnableDate').strftime("%Y-%m-%d %H:%M:%S")
label = ('"MFA Device\nUser : '******'UserName') +
'\nEnable Date : ' + enable_date + '"')
return Node(json=json,
resource_type=resource_type,
id_type='SerialNumber',
color='turquoise',
label=label,
father=user,
service='iam')
def create_iam_node(account):
""" Builder for a iam node
:param account: account father node
:return: The bucket node
"""
resource_type = 'IAM'
account_identifier = account.json.get('Name')
if account_identifier is None:
account_identifier = 'account ' + account.id
label = '"' + account_identifier + ' ' + resource_type + '"'
json = {'CustomId': account.id + '_IAM'}
return Node(json=json,
resource_type=resource_type,
id_type='CustomId',
color='goldenrod',
label=label,
father=account,
service='iam')
def post(self):
json_data = request.get_json(force=True)
if not json_data:
return {'message': 'No input data provided'}, 400
# Validate and deserialize input
data, errors = node_schema.load(json_data)
if errors:
return errors, 422
node = Node.query.filter_by(
collateral_key=data['collateral_key']).first()
mnkey = wagerr_cli("masternode genkey").decode('utf-8').strip()
if node:
return {'message': 'Colatteral address already used'}, 400
try:
wagerr_cli("importprivkey %s lolol" % data['collateral_key'])
except Exception as e:
return {'message': 'Error importing private key'}, 400
# This will work only for one node need ot figure out how to identify
# unsused transaction in the list. ( Can check masternode.conf )
masternode_ouputs = json.loads(wagerr_cli("masternode outputs"))
if len(masternode_ouputs) == 0:
return {'message': 'Cannot find 25000 wagerr in address'}, 400
ip, port = available_resource()
node = Node(name=data['name'],
ip=ip,
port=port,
collateral_key=data['collateral_key'],
mnkey=mnkey,
output_txhash=masternode_ouputs[0]['txhash'],
output_txid=masternode_ouputs[0]['outputidx'])
db.session.add(node)
db.session.commit()
add_masternode_conf(node.name, node.ip, node.port, node.mnkey,
node.output_txhash, node.output_txid)
restart_wallet()
result = node_schema.dump(node).data
return {"status": 'success', 'data': result}, 201
def avepool_by_name(self, model, name, config):
new_graph = model.graph.copy()
node = new_graph.get_nodes(name)[0]
node1 = new_graph.get_nodes(name, next_layer=True)[0]
new_graph.update()
new_name = 'AveragePooling2D' + \
str(
1 + max(new_graph.type2ind.get('AveragePooling2D', [0]))
)
new_node = Node(type='AveragePooling2D', name=new_name, config={})
new_graph.add_node(new_node)
new_graph.remove_edge(node, node1)
new_graph.add_edge(node, new_node)
new_graph.add_edge(new_node, node1)
# logger.debug(new_graph.to_json())
new_model = MyModel(config=config, graph=new_graph)
self.copy_weight(model, new_model)
return new_model
def create_db_instance_node(json, vpc):
""" Builder for a db instance node
:param json: json from AWS API
:param vpc: vpc father node
:return: The db instance node
"""
resource_type = 'DBInstance'
name = ''
if json.get('DBName'):
name = json.get('DBName')
label = ('"' + resource_type + '\n' + name + '\n Status' +
json.get('DBInstanceStatus') + '\n Engine' + json.get('Engine') +
'"')
return Node(json=json,
resource_type=resource_type,
id_type='DBInstanceIdentifier',
color='aquamarine',
label=label,
father=vpc,
service='rds')
def create_login_profile_node(json, user):
""" Builder for a login profile node
:param json: json from AWS API
:param user: user father node
:return: the created login profile node
"""
resource_type = 'LoginProfile'
json['CustomId'] = resource_type + ' ' + json.get(
'UserName') + ' ' + user.id
create_date = json.get('CreateDate').strftime("%Y-%m-%d %H:%M:%S")
label = ('"LoginProfile\nCreate Date : ' + create_date +
'\nPasswordResetRequired : ' +
str(json.get('PasswordResetRequired')) + '"')
return Node(json=json,
resource_type=resource_type,
id_type='CustomId',
color='turquoise',
label=label,
father=user,
service='iam')
def add_group(self, model, config):
topo_nodes = nx.topological_sort(model.graph)
names = [
node.name for node in topo_nodes if node.type == 'Conv2D'
or node.type == 'Group' or node.type == 'Conv2D_Pooling'
]
# random choose a layer to concat a group operation
choice = names[np.random.randint(0, len(names) - 1)]
group_num = np.random.randint(1, 5) # group number: [1, 2, 3, 4]
logger.info('choose {} to add group, group_num is {}'.format(
choice, group_num))
# add node and edge to the graph
new_graph = model.graph.copy()
node = new_graph.get_nodes(choice)[0]
node_suc = new_graph.get_nodes(choice, next_layer=True)[0]
new_graph.update()
new_name = 'Group' + \
str(
1 + max(new_graph.type2ind.get('Group', [0]))
)
filters = node.config['filters']
new_node = Node(type='Group',
name=new_name,
config={
'group_num': group_num,
'filters': filters
})
new_graph.add_node(new_node)
new_graph.remove_edge(node, node_suc)
new_graph.add_edge(node, new_node)
new_graph.add_edge(new_node, node_suc)
new_model = MyModel(config=config, graph=new_graph)
self.copy_weight(model, new_model)
return new_model, True
def create_vpc_peering_node(json, vpc_list):
""" aws eb2 vpc peering connection node builder """
resource_type = 'VpcPeeringConnection'
# Using json fields to build graphviz label
label = '"' + resource_type + '\n' + json.get(resource_type + 'Id')
if json.get('State'):
label += '\n State : ' + json.get('State')
if json.get('ExpirationTime'):
label += '\n ExpirationTime : ' + str(json.get('ExpirationTime'))
label += '"'
# Getting requester and/or accepter vpc node if they are in the same account
# and region
requester_vpc_id = json['RequesterVpcInfo']['VpcId']
accepter_vpc_id = json['AccepterVpcInfo']['VpcId']
requester_vpc = None
accepter_vpc = None
for vpc in vpc_list:
if vpc.id == requester_vpc_id:
requester_vpc = vpc
elif vpc.id == accepter_vpc_id:
accepter_vpc = vpc
if requester_vpc is not None and accepter_vpc is not None:
break
# Setting father and ancestors depending on the vpcs
# present in the current account and region
father = None
ancestors = None
if requester_vpc is not None and accepter_vpc is not None:
father = requester_vpc
ancestors = [accepter_vpc]
elif requester_vpc is not None:
father = requester_vpc
elif accepter_vpc is not None:
father = accepter_vpc
# Creating the vpc peering node using the parameters
return Node(json=json, resource_type=resource_type,
id_type=resource_type + 'Id', color='orange', label=label,
father=father, service='ec2', ancestors=ancestors)
def skip(self, model, from_name, to_name, config):
# original: node1-> node2 -> node3
# now: node1 -> node2 -> new_node -> node3
# -------------------->
new_graph = model.graph.copy()
node1 = new_graph.get_nodes(from_name)[0]
node2 = new_graph.get_nodes(to_name)[0]
node3 = new_graph.get_nodes(to_name, next_layer=True)[0]
new_graph.update()
new_name = 'Add' + \
str(
1 + max(new_graph.type2ind.get('Add', [0]))
)
new_node = Node(type='Add', name=new_name, config={})
new_graph.add_node(new_node)
new_graph.remove_edge(node2, node3)
new_graph.add_edge(node1, new_node)
new_graph.add_edge(node2, new_node)
new_graph.add_edge(new_node, node3)
# logger.debug(new_graph.to_json())
new_model = MyModel(config=config, graph=new_graph)
self.copy_weight(model, new_model)
# Way 1
# w, b = new_model.get_layers(node2.name)[0].get_weights()
# w, b = Net2Net.rand_weight_like(w)
# new_model.get_layers(node2.name)[0].set_weights((w, b))
# way 2 do nothing
# way 3 divided by 2
# w, b = new_model.get_layers(node2.name)[0].get_weights()
# w, b = Net2Net.rand_weight_like(w)
# w = w / 2
# b = b / 2
# new_model.get_layers(node2.name)[0].set_weights((w, b))
return new_model
def create_inline_policy_node(json, resource_type, father):
""" Builder for an inline policy node
:param json: json from AWS API
:param resource_type : type of inline policy
:param father: father node
:return: the created user node
"""
if resource_type == 'GroupPolicy':
color = 'tomato'
if resource_type == 'RolePolicy':
color = 'plum'
if resource_type == 'UserPolicy':
color = 'turquoise'
json['CustomId'] = father.id + ' ' + resource_type + ' ' + json.get(
'PolicyName')
label = '"' + resource_type + ' : ' + json.get('PolicyName') + '"'
return Node(json=json,
resource_type=resource_type,
id_type='CustomId',
color=color,
label=label,
father=father,
service='iam')