def main(): assert os.path.exists(DATA_PATH) images = util.get_image_names(DATA_PATH) num_images = len(images) images_with_path = [DATA_PATH + i + '.jpg' for i in images] ##load labels labels = util.get_attributes('../data/pubfig_attributes.txt', images) labels = labels.as_matrix() fig, ax = plt.subplots() x = [] y = [] for i in xrange(labels.shape[1]): print 'Label %d of %d' % (i + 1, labels.shape[1]) x.append(i) y.append(np.sum((labels[:, i] > 0.5))) print LABELS[i], y[i] return print len(x), len(y) plt.bar(x, y, color="blue") # ax.set_xlim([0.0,20.0]) ax.set_xticks(range(len(LABELS))) ax.set_xticklabels(LABELS, rotation=90, size=8) # ax.get_xaxis().majorTicks[2].set_pad(50) plt.gcf().subplots_adjust(bottom=0.30) plt.show() plt.savefig('distrib.png')
VERBOSE = True if sys.argv[1] == 'eval': SET = 'eval' if sys.argv[1] == 'dev': SET = 'dev' DATA_PATH = "../data/%s_set/images_cropped"%(SET) ATTR_PATH = "../data/pubfig_attributes.txt" OUTPUT_PATH = "../data/" print 'Loading mean image...' mean_image = np.load("../data/mean_image.npy").astype(np.int) print 'Loading labels....' images = util.get_image_names(DATA_PATH, remove_extension=True) labels = util.consolidate_labels(util.get_attributes(ATTR_PATH, images)) print 'Loading image list....' images = [os.path.join(DATA_PATH, filename) for filename in util.get_image_names(DATA_PATH, remove_extension=False)] in_db = lmdb.open(os.path.join(OUTPUT_PATH, 'image-%s-lmdb'%(SET)), map_size=int(1e12)) labels_db = lmdb.open(os.path.join(OUTPUT_PATH, 'labels-%s-lmdb'%(SET)), map_size=int(1e12)) with in_db.begin(write=True) as in_txn: for in_idx, in_ in enumerate(images): if in_idx % 100 == 0: print 'Image %d/%d done.'%(in_idx, len(images)) # load image: # - as np.uint8 {0, ..., 255} # - in BGR (switch from RGB)
def main(): if len(sys.argv) == 1: mode = "PLOT" else: if sys.argv[1] == "PLOT" or sys.argv[1] == "EXTRACT": mode = sys.argv[1] else: print 'Usage: python vgg_visualizer.py EXTRACT|PLOT' sys.exit(1) if mode == "EXTRACT": assert os.path.exists(DATA_PATH) assert os.path.exists(WEIGHTS_PATH) assert os.path.exists(SOLVER_PATH) caffe.set_mode_gpu() images = util.get_image_names(DATA_PATH) start_idx = 0 end_idx = len(images) # TODO: Use command line arguments to split into batches if necessary # start_idx = int(sys.argv[1]) # end_idx = int(sys.argv[2]) images = images[start_idx:end_idx] if len(images) == 0: return if not os.path.exists(OUTPUT_PATH + 'outs_fc7'): os.makedirs(OUTPUT_PATH + 'outs_fc7') with open(OUTPUT_PATH + 'outs_fc7/image_list.dat', 'w+') as fp: cp.dump(images, fp) images_with_path = [DATA_PATH + i + '.jpg' for i in images] if len(images) % BATCH_SIZE != 0: images_with_path = images_with_path + [images_with_path[-1]] * ( BATCH_SIZE - (len(images) % BATCH_SIZE)) net = caffe.Net(SOLVER_PATH, WEIGHTS_PATH, caffe.TEST) transformer = caffe.io.Transformer( {'data': net.blobs['data'].data.shape}) transformer.set_transpose('data', (2, 0, 1)) data_blob_shape = net.blobs['data'].data.shape data_blob_shape = list(data_blob_shape) net.blobs['data'].reshape(BATCH_SIZE, data_blob_shape[1], data_blob_shape[2], data_blob_shape[3]) out_fc7 = np.zeros((len(images), 4096)) for i in xrange(len(images_with_path) / BATCH_SIZE): net_forward(net, transformer, images_with_path[i * BATCH_SIZE:(i + 1) * BATCH_SIZE], out_fc7, i, BATCH_SIZE, len(images), 'blob_%d_%d' % (start_idx, end_idx)) np.save(OUTPUT_PATH + 'outs_fc7/blob_%d_%d.dat' % (start_idx, end_idx), out_fc7) print '[blob_%d_%d] done.' % (start_idx, end_idx) else: # Visualize with open(OUTPUT_PATH + 'outs_fc7/image_list.dat', 'rb') as fp: images = cp.load(fp) start_idx = 0 end_idx = len(images) # TODO: Use command line arguments to split into batches if necessary # start_idx = int(sys.argv[1]) # end_idx = int(sys.argv[2]) out_fc7 = np.load(OUTPUT_PATH + 'outs_fc7/blob_%d_%d.dat.npy' % (start_idx, end_idx)) labels = util.get_attributes('../data/pubfig_attributes.txt', images) print labels.shape print out_fc7.shape labels = labels.as_matrix() plot_tsne(out_fc7, labels)
def _interfaceParse___iface_attributes(config, check_disabled): iface_list = util.get_attributes(config)[0] # if iface isn`t enable and unused if iface_list: iface_dict = { 'shutdown': 'no', 'vlans': [], 'cdp': 'yes', 'dhcp_snoop': { 'mode': 'untrust' }, 'arp_insp': { 'mode': 'untrust' }, 'storm control': {}, 'port-security': {}, 'ipv6': {} } vlan_num = Word(nums + '-') + ZeroOrMore(Suppress(',') + Word(nums + '-')) parse_description = Suppress('description ') + restOfLine parse_type = Suppress('switchport mode ') + restOfLine parse_port_sec = Suppress('switchport port-security ') + restOfLine parse_stp_port = Suppress('spanning-tree ') + restOfLine parse_dhcp_snoop = Suppress('ip dhcp snooping ') + restOfLine parse_arp_insp = Suppress('ip arp inspection ') + restOfLine parse_source_guard = Suppress('ip verify source ') + restOfLine parse_arp_proxy_iface = Optional( Word(alphas)) + Suppress('ip proxy-arp') parse_vlans = Suppress('switchport ') + Suppress( MatchFirst('access vlan ' + ('trunk allowed vlan ' + Optional('add ')))) + vlan_num class Storm: def __init__(self): self.dct = {'type': []} def new_line(self, line): parse_storm = Suppress('storm-control ') + restOfLine try: self.storm_line = parse_storm.parseString( line).asList()[-1] self.level_info() self.action_info() self.type_info() except ParseException: pass @catch_exception def parse_level(self): parse_level = Word(alphas) + Suppress('level ') + restOfLine value = parse_level.parseString(self.storm_line).asList() if 'level' in self.dct: self.dct['level'].append(value) else: self.dct['level'] = [value] @catch_exception def parse_action(self): action = Suppress('action ') + Word(alphas) self.action = utill(action, self.storm_line) @catch_exception def parse_type(self): type = Word(alphas) + Suppress( Optional("include")) + Word(alphas) self.type = utill(type, self.storm_line) @catch_exception1 def action_info(self): self.parse_action() self.dct['action'] = self.action @catch_exception1 def type_info(self): self.parse_type() for each in self.type: if each not in self.dct['type'] and each in [ 'broadcast', 'multicast', 'unicast' ]: self.dct['type'].append(each) @catch_exception1 def level_info(self): self.parse_level() cl_storm = Storm() # Reserved options list is using due to 'shutdown' option is usually located at the end of the list, so it breaks cycle if interface is shutdown and function speed increases for option in iface_list[::-1]: cl_storm.new_line(option) iface_dict['storm control'] = cl_storm.dct if option == 'shutdown': if check_disabled: iface_dict['shutdown'] = 'yes' pass else: iface_dict = {'shutdown': 'yes'} break if option == 'switchport nonegotiate': iface_dict['dtp'] = 'no' continue if option == 'no cdp enable': iface_dict['cdp'] = 'no' continue if option == 'no mop enabled': iface_dict['mop'] = 'no' continue elif option == 'mop enabled': iface_dict['mop'] = 'yes' continue try: vlan_add = parse_vlans.parseString(option).asList() for unit in vlan_add: if '-' in unit: range_units = unit.split('-') range_list = [ i for i in range(int(range_units[0]), int(range_units[1]) + 1) ] iface_dict['vlans'].extend(range_list) else: iface_dict['vlans'].append(int(unit)) continue except ParseException: pass try: iface_dict['description'] = parse_description.parseString( option).asList()[-1] continue except ParseException: pass try: iface_dict['type'] = parse_type.parseString( option).asList()[-1] continue except ParseException: pass try: port_sec = parse_port_sec.parseString(option).asList()[-1] iface_dict[ 'port-security'] = parsing_checks.port_security.__ifaceAttributes___port_sec_parse( port_sec, iface_dict['port-security']) continue except ParseException: pass try: dhcp_snoop = parse_dhcp_snoop.parseString(option).asList()[-1] iface_dict[ 'dhcp_snoop'] = parsing_checks.ip_iface.__ifaceAttributes___ip_parse( dhcp_snoop, iface_dict['dhcp_snoop']) continue except ParseException: pass try: arp_insp = parse_arp_insp.parseString(option).asList()[-1] iface_dict[ 'arp_insp'] = parsing_checks.ip_iface.__ifaceAttributes___ip_parse( arp_insp, iface_dict['arp_insp']) # making a str:str instead of str:list # see issue #5 if 'mode' in iface_dict['arp_insp']: iface_dict['arp_insp']['mode'] = iface_dict['arp_insp'][ 'mode'][0] continue except ParseException: pass try: stp_port = parse_stp_port.parseString(option).asList()[-1] iface_dict['stp'] = stp_port continue except ParseException: pass try: source_guard = parse_source_guard.parseString( option).asList()[-1] iface_dict['source_guard'] = source_guard continue except ParseException: pass try: ipv6 = parse_ipv6.parseString(option).asList()[-1] __ifaceAttributes___ipv6_parse(ipv6, iface_dict['ipv6']) continue except ParseException: pass try: arp_proxy_iface = parse_arp_proxy_iface.parseString( option).asList()[-1] iface_dict['arp_proxy'] = arp_proxy_iface continue except ParseException: pass return iface_dict else: return {'unknown_iface': 1}
def _globalParse___line_attributes(config): line_list, next_line = util.get_attributes(config) line_dict = {'log_syng': 'no', 'no_exec': 'no', 'access-class': {}} parse_login_type = Suppress('login ' + Optional('authentication ')) + restOfLine parse_exec_timeout = Suppress('exec-timeout ') + restOfLine parse_pass_type = Suppress('password ') + restOfLine parse_privilege = Suppress('privilege level ') + restOfLine parse_transp_in = Suppress('transport input ') + restOfLine parse_transp_out = Suppress('transport output ') + restOfLine parse_rotary = Suppress('rotary ') + restOfLine parse_access_class = Suppress('access-class') + Word( alphas + '-') + MatchFirst(['in', 'out']) + Suppress( Optional(restOfLine)) for option in line_list: if option == 'logging synchronous': line_dict['log_syng'] = 'yes' continue if option == 'no exec': line_dict['no_exec'] = 'yes' continue try: item = parse_exec_timeout.parseString(option).asList()[-1] item = item.split() if len(item) == 2: line_dict['exec_timeout'] = int(item[0]) + int(item[1]) / 60 else: line_dict['exec_timeout'] = int(item[0]) continue except ParseException: pass try: line_dict['login_type'] = parse_login_type.parseString( option).asList()[-1] continue except ParseException: pass try: line_dict['pass_type'] = parse_pass_type.parseString( option).asList()[-1] continue except ParseException: pass try: line_dict['privilege'] = parse_privilege.parseString( option).asList()[-1] continue except ParseException: pass try: line_dict['transp_in'] = parse_transp_in.parseString( option).asList()[-1] continue except ParseException: pass try: line_dict['transp_out'] = parse_transp_out.parseString( option).asList()[-1] continue except ParseException: pass try: line_dict['rotary'] = parse_rotary.parseString(option).asList()[-1] continue except ParseException: pass try: item = parse_access_class.parseString(option).asList() line_dict['access-class']['name'] = item[0] line_dict['access-class']['type'] = item[-1] continue except ParseException: pass return line_dict, next_line