def main():
assert os.path.exists(DATA_PATH)
images = util.get_image_names(DATA_PATH)
num_images = len(images)
images_with_path = [DATA_PATH + i + '.jpg' for i in images]
##load labels
labels = util.get_attributes('../data/pubfig_attributes.txt', images)
labels = labels.as_matrix()
fig, ax = plt.subplots()
x = []
y = []
for i in xrange(labels.shape[1]):
print 'Label %d of %d' % (i + 1, labels.shape[1])
x.append(i)
y.append(np.sum((labels[:, i] > 0.5)))
print LABELS[i], y[i]
return
print len(x), len(y)
plt.bar(x, y, color="blue")
# ax.set_xlim([0.0,20.0])
ax.set_xticks(range(len(LABELS)))
ax.set_xticklabels(LABELS, rotation=90, size=8)
# ax.get_xaxis().majorTicks[2].set_pad(50)
plt.gcf().subplots_adjust(bottom=0.30)
plt.show()
plt.savefig('distrib.png')
VERBOSE = True
if sys.argv[1] == 'eval':
SET = 'eval'
if sys.argv[1] == 'dev':
SET = 'dev'
DATA_PATH = "../data/%s_set/images_cropped"%(SET)
ATTR_PATH = "../data/pubfig_attributes.txt"
OUTPUT_PATH = "../data/"
print 'Loading mean image...'
mean_image = np.load("../data/mean_image.npy").astype(np.int)
print 'Loading labels....'
images = util.get_image_names(DATA_PATH, remove_extension=True)
labels = util.consolidate_labels(util.get_attributes(ATTR_PATH, images))
print 'Loading image list....'
images = [os.path.join(DATA_PATH, filename) for filename in util.get_image_names(DATA_PATH, remove_extension=False)]
in_db = lmdb.open(os.path.join(OUTPUT_PATH, 'image-%s-lmdb'%(SET)), map_size=int(1e12))
labels_db = lmdb.open(os.path.join(OUTPUT_PATH, 'labels-%s-lmdb'%(SET)), map_size=int(1e12))
with in_db.begin(write=True) as in_txn:
for in_idx, in_ in enumerate(images):
if in_idx % 100 == 0:
print 'Image %d/%d done.'%(in_idx, len(images))
# load image:
# - as np.uint8 {0, ..., 255}
# - in BGR (switch from RGB)
def main():
if len(sys.argv) == 1:
mode = "PLOT"
else:
if sys.argv[1] == "PLOT" or sys.argv[1] == "EXTRACT":
mode = sys.argv[1]
else:
print 'Usage: python vgg_visualizer.py EXTRACT|PLOT'
sys.exit(1)
if mode == "EXTRACT":
assert os.path.exists(DATA_PATH)
assert os.path.exists(WEIGHTS_PATH)
assert os.path.exists(SOLVER_PATH)
caffe.set_mode_gpu()
images = util.get_image_names(DATA_PATH)
start_idx = 0
end_idx = len(images)
# TODO: Use command line arguments to split into batches if necessary
# start_idx = int(sys.argv[1])
# end_idx = int(sys.argv[2])
images = images[start_idx:end_idx]
if len(images) == 0:
return
if not os.path.exists(OUTPUT_PATH + 'outs_fc7'):
os.makedirs(OUTPUT_PATH + 'outs_fc7')
with open(OUTPUT_PATH + 'outs_fc7/image_list.dat', 'w+') as fp:
cp.dump(images, fp)
images_with_path = [DATA_PATH + i + '.jpg' for i in images]
if len(images) % BATCH_SIZE != 0:
images_with_path = images_with_path + [images_with_path[-1]] * (
BATCH_SIZE - (len(images) % BATCH_SIZE))
net = caffe.Net(SOLVER_PATH, WEIGHTS_PATH, caffe.TEST)
transformer = caffe.io.Transformer(
{'data': net.blobs['data'].data.shape})
transformer.set_transpose('data', (2, 0, 1))
data_blob_shape = net.blobs['data'].data.shape
data_blob_shape = list(data_blob_shape)
net.blobs['data'].reshape(BATCH_SIZE, data_blob_shape[1],
data_blob_shape[2], data_blob_shape[3])
out_fc7 = np.zeros((len(images), 4096))
for i in xrange(len(images_with_path) / BATCH_SIZE):
net_forward(net, transformer,
images_with_path[i * BATCH_SIZE:(i + 1) * BATCH_SIZE],
out_fc7, i, BATCH_SIZE, len(images),
'blob_%d_%d' % (start_idx, end_idx))
np.save(OUTPUT_PATH + 'outs_fc7/blob_%d_%d.dat' % (start_idx, end_idx),
out_fc7)
print '[blob_%d_%d] done.' % (start_idx, end_idx)
else:
# Visualize
with open(OUTPUT_PATH + 'outs_fc7/image_list.dat', 'rb') as fp:
images = cp.load(fp)
start_idx = 0
end_idx = len(images)
# TODO: Use command line arguments to split into batches if necessary
# start_idx = int(sys.argv[1])
# end_idx = int(sys.argv[2])
out_fc7 = np.load(OUTPUT_PATH + 'outs_fc7/blob_%d_%d.dat.npy' %
(start_idx, end_idx))
labels = util.get_attributes('../data/pubfig_attributes.txt', images)
print labels.shape
print out_fc7.shape
labels = labels.as_matrix()
plot_tsne(out_fc7, labels)
def _interfaceParse___iface_attributes(config, check_disabled):
iface_list = util.get_attributes(config)[0]
# if iface isn`t enable and unused
if iface_list:
iface_dict = {
'shutdown': 'no',
'vlans': [],
'cdp': 'yes',
'dhcp_snoop': {
'mode': 'untrust'
},
'arp_insp': {
'mode': 'untrust'
},
'storm control': {},
'port-security': {},
'ipv6': {}
}
vlan_num = Word(nums +
'-') + ZeroOrMore(Suppress(',') + Word(nums + '-'))
parse_description = Suppress('description ') + restOfLine
parse_type = Suppress('switchport mode ') + restOfLine
parse_port_sec = Suppress('switchport port-security ') + restOfLine
parse_stp_port = Suppress('spanning-tree ') + restOfLine
parse_dhcp_snoop = Suppress('ip dhcp snooping ') + restOfLine
parse_arp_insp = Suppress('ip arp inspection ') + restOfLine
parse_source_guard = Suppress('ip verify source ') + restOfLine
parse_arp_proxy_iface = Optional(
Word(alphas)) + Suppress('ip proxy-arp')
parse_vlans = Suppress('switchport ') + Suppress(
MatchFirst('access vlan ' +
('trunk allowed vlan ' + Optional('add ')))) + vlan_num
class Storm:
def __init__(self):
self.dct = {'type': []}
def new_line(self, line):
parse_storm = Suppress('storm-control ') + restOfLine
try:
self.storm_line = parse_storm.parseString(
line).asList()[-1]
self.level_info()
self.action_info()
self.type_info()
except ParseException:
pass
@catch_exception
def parse_level(self):
parse_level = Word(alphas) + Suppress('level ') + restOfLine
value = parse_level.parseString(self.storm_line).asList()
if 'level' in self.dct:
self.dct['level'].append(value)
else:
self.dct['level'] = [value]
@catch_exception
def parse_action(self):
action = Suppress('action ') + Word(alphas)
self.action = utill(action, self.storm_line)
@catch_exception
def parse_type(self):
type = Word(alphas) + Suppress(
Optional("include")) + Word(alphas)
self.type = utill(type, self.storm_line)
@catch_exception1
def action_info(self):
self.parse_action()
self.dct['action'] = self.action
@catch_exception1
def type_info(self):
self.parse_type()
for each in self.type:
if each not in self.dct['type'] and each in [
'broadcast', 'multicast', 'unicast'
]:
self.dct['type'].append(each)
@catch_exception1
def level_info(self):
self.parse_level()
cl_storm = Storm()
# Reserved options list is using due to 'shutdown' option is usually located at the end of the list, so it breaks cycle if interface is shutdown and function speed increases
for option in iface_list[::-1]:
cl_storm.new_line(option)
iface_dict['storm control'] = cl_storm.dct
if option == 'shutdown':
if check_disabled:
iface_dict['shutdown'] = 'yes'
pass
else:
iface_dict = {'shutdown': 'yes'}
break
if option == 'switchport nonegotiate':
iface_dict['dtp'] = 'no'
continue
if option == 'no cdp enable':
iface_dict['cdp'] = 'no'
continue
if option == 'no mop enabled':
iface_dict['mop'] = 'no'
continue
elif option == 'mop enabled':
iface_dict['mop'] = 'yes'
continue
try:
vlan_add = parse_vlans.parseString(option).asList()
for unit in vlan_add:
if '-' in unit:
range_units = unit.split('-')
range_list = [
i for i in range(int(range_units[0]),
int(range_units[1]) + 1)
]
iface_dict['vlans'].extend(range_list)
else:
iface_dict['vlans'].append(int(unit))
continue
except ParseException:
pass
try:
iface_dict['description'] = parse_description.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
iface_dict['type'] = parse_type.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
port_sec = parse_port_sec.parseString(option).asList()[-1]
iface_dict[
'port-security'] = parsing_checks.port_security.__ifaceAttributes___port_sec_parse(
port_sec, iface_dict['port-security'])
continue
except ParseException:
pass
try:
dhcp_snoop = parse_dhcp_snoop.parseString(option).asList()[-1]
iface_dict[
'dhcp_snoop'] = parsing_checks.ip_iface.__ifaceAttributes___ip_parse(
dhcp_snoop, iface_dict['dhcp_snoop'])
continue
except ParseException:
pass
try:
arp_insp = parse_arp_insp.parseString(option).asList()[-1]
iface_dict[
'arp_insp'] = parsing_checks.ip_iface.__ifaceAttributes___ip_parse(
arp_insp, iface_dict['arp_insp'])
# making a str:str instead of str:list
# see issue #5
if 'mode' in iface_dict['arp_insp']:
iface_dict['arp_insp']['mode'] = iface_dict['arp_insp'][
'mode'][0]
continue
except ParseException:
pass
try:
stp_port = parse_stp_port.parseString(option).asList()[-1]
iface_dict['stp'] = stp_port
continue
except ParseException:
pass
try:
source_guard = parse_source_guard.parseString(
option).asList()[-1]
iface_dict['source_guard'] = source_guard
continue
except ParseException:
pass
try:
ipv6 = parse_ipv6.parseString(option).asList()[-1]
__ifaceAttributes___ipv6_parse(ipv6, iface_dict['ipv6'])
continue
except ParseException:
pass
try:
arp_proxy_iface = parse_arp_proxy_iface.parseString(
option).asList()[-1]
iface_dict['arp_proxy'] = arp_proxy_iface
continue
except ParseException:
pass
return iface_dict
else:
return {'unknown_iface': 1}
def _globalParse___line_attributes(config):
line_list, next_line = util.get_attributes(config)
line_dict = {'log_syng': 'no', 'no_exec': 'no', 'access-class': {}}
parse_login_type = Suppress('login ' +
Optional('authentication ')) + restOfLine
parse_exec_timeout = Suppress('exec-timeout ') + restOfLine
parse_pass_type = Suppress('password ') + restOfLine
parse_privilege = Suppress('privilege level ') + restOfLine
parse_transp_in = Suppress('transport input ') + restOfLine
parse_transp_out = Suppress('transport output ') + restOfLine
parse_rotary = Suppress('rotary ') + restOfLine
parse_access_class = Suppress('access-class') + Word(
alphas + '-') + MatchFirst(['in', 'out']) + Suppress(
Optional(restOfLine))
for option in line_list:
if option == 'logging synchronous':
line_dict['log_syng'] = 'yes'
continue
if option == 'no exec':
line_dict['no_exec'] = 'yes'
continue
try:
item = parse_exec_timeout.parseString(option).asList()[-1]
item = item.split()
if len(item) == 2:
line_dict['exec_timeout'] = int(item[0]) + int(item[1]) / 60
else:
line_dict['exec_timeout'] = int(item[0])
continue
except ParseException:
pass
try:
line_dict['login_type'] = parse_login_type.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
line_dict['pass_type'] = parse_pass_type.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
line_dict['privilege'] = parse_privilege.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
line_dict['transp_in'] = parse_transp_in.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
line_dict['transp_out'] = parse_transp_out.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
line_dict['rotary'] = parse_rotary.parseString(option).asList()[-1]
continue
except ParseException:
pass
try:
item = parse_access_class.parseString(option).asList()
line_dict['access-class']['name'] = item[0]
line_dict['access-class']['type'] = item[-1]
continue
except ParseException:
pass
return line_dict, next_line