def __init__(self, phase, num_categories, shuffle=False): # Load the complete modelnet40 training data # Training data is dispersed over 5 files, # load them all into data (point clouds) and labels points = [] labels = [] i = 0 while True: filename = os.path.join("modelnet40", "ply_data_%s%i.h5" % (phase, i)) if not os.path.isfile(filename): break p, l = utils.load_h5(filename) points.extend(np.array(p)) labels.extend(np.squeeze(np.array(l), 1)) i += 1 self.num_examples = len(points) self._index_in_epoch = 0 self._points = np.array(points) self._labels = np.zeros((self.num_examples, num_categories)) self._labels[np.arange(self.num_examples), np.array(labels)] = 1 self._shuffle = shuffle self._epoch_complete = False if self._shuffle: self._shuffle_data()
def ipheadertask(filelist): j = 1 for fullname in filelist: print("Loading filenr: {}".format(j)) load_dir, filename = os.path.split(fullname) df = utils.load_h5(load_dir, filename) frames = df['bytes'].values for i, frame in enumerate(frames): p = np.fromstring(frame, dtype=np.uint8) if p[14] != 69: print("IP Header length not 20! in file {0}".format(filename)) j += 1
seed = 0 num_headers = 16 dirs = ["C:/Users/salik/Documents/Data/LinuxChrome/{}/".format(num_headers), "C:/Users/salik/Documents/Data/WindowsFirefox/{}/".format(num_headers), "C:/Users/salik/Documents/Data/WindowsChrome/{}/".format(num_headers), "C:/Users/salik/Documents/Data/WindowsSalik/{}/".format(num_headers), "C:/Users/salik/Documents/Data/WindowsAndreas/{}/".format(num_headers)] # dirs = ["E:/Data/h5/https/", "E:/Data/h5/netflix/"] # dirs = ["C:/Users/salik/Documents/Data/WindowsAndreas/{}/".format(num_headers)] # step 1: get the data dataframes = [] num_examples = 0 for dir in dirs: for fullname in glob.iglob(dir + '*.h5'): filename = os.path.basename(fullname) df = utils.load_h5(dir, filename) dataframes.append(df) num_examples = len(df.values) # create one large dataframe data = pd.concat(dataframes) data.sample(frac=1, random_state=seed).reset_index(drop=True) num_rows = data.shape[0] columns = data.columns print(columns) # step 3: get features (x) and scale the features # get x and convert it to numpy array # x = da.getbytes(data, 1460) standard_scaler = StandardScaler() x = da.getbytes(data, num_headers*54) x_std = standard_scaler.fit_transform(x)
def train_model(): learning_rate = 0.0001 batch_size = 128 c_dim = 1 n_batch = 10000 // batch_size x_input = tf.placeholder(tf.float32, [None, None, None, c_dim], name='x_input') y_label2 = tf.placeholder(tf.float32, [None, None, None, c_dim], name='y_label2') x1_bic_add = tf.placeholder(tf.float32, [None, None, None, c_dim], name='x1_bic_add') output = define_model(x_input, x1_bic_add) mse_loss = tf.reduce_mean(tf.square(y_label2 - output)) train = tf.train.GradientDescentOptimizer(learning_rate).minimize(mse_loss) saver = tf.train.Saver(max_to_keep=4) tf.add_to_collection("predict", output) with tf.Session() as sess: init = tf.global_variables_initializer() sess.run(init) # load train set x1d = load_h5('Y_dataset.h5', 'train_cut_l') x2d = load_h5('Y_dataset.h5', 'train_cut_h') x1_b = load_h5('Y_dataset.h5', 'train_cut_b') x1_data = x1d[:10000, :, :, :] / 255 x2_data = x2d[:10000, :, :, :] / 255 x1_bic = x1_b[:10000, :, :, :] / 255 print(x1_data.shape, x2_data.shape, x1_bic.shape) counter = 0 for i in range(2001): for idx in range(0, n_batch): batch_images = x1_data[idx * batch_size:(idx + 1) * batch_size] batch_bic = x1_bic[idx * batch_size:(idx + 1) * batch_size] batch_labels = x2_data[idx * batch_size:(idx + 1) * batch_size] sess.run(train, feed_dict={ x_input: batch_images, y_label2: batch_labels, x1_bic_add: batch_bic }) counter += 1 if counter % 50 == 0: print( 'Epoh', i, 'n_batch', idx, 'Train loss:', sess.run(mse_loss, feed_dict={ x_input: batch_images, y_label2: batch_labels, x1_bic_add: batch_bic })) rands(x1_data, x2_data, x1_bic) if i % 10 == 0: saver.save(sess, r'./checkpoint/' + "model_conv/my-model", global_step=i) print("save the model")
"--minclass", type=int, nargs="?", dest='minclass', help='Minimum Number of Classes') parser.add_argument("-w", "--maxclass", type=int, nargs="?", dest='maxclass', help='Maximum Number of Classes') parser.add_argument("-r", "--repeat", type=bool, nargs="?", dest='repeat', help='Allow Repeated Data Among Clients') args = parser.parse_args() trainX, trainY = load_h5("cifar/train_data.h5") labels = load_labels("cifar/labels.h5") separated_data = separate_data_by_class(trainX, trainY, labels) generate_client_dataset_files(dataset=separated_data, directory=args.name, n_clients=args.total, n_samples_min=args.minsample, n_samples_max=args.maxsample, n_classes_min=args.minclass, n_classes_max=args.maxclass, no_repeat=not (args.repeat))
# coding: utf-8 # In[69]: import tensorflow as tf import utils import numpy as np import os # In[71]: # load training data and labels data0 = utils.load_h5("ply_data_train0.h5") data1 = utils.load_h5("ply_data_train1.h5") data2 = utils.load_h5("ply_data_train2.h5") data3 = utils.load_h5("ply_data_train3.h5") data4 = utils.load_h5("ply_data_train4.h5") # train_data = data0[0] # print(np.shape(train_data)) # train_labels = data0[1] # catagory_names = utils.get_category_names() # print(np.shape(train_labels)) # In[72]: # aggregate training data, training label train_data = np.append(data0[0], data1[0], axis=0) train_data = np.append(train_data, data2[0], axis=0) train_data = np.append(train_data, data3[0], axis=0)
data = f(SELECTED_SUBSET) match_links = link(SELECTED_SUBSET) #print("Match links:", match_links) print("Precompute SIFT, MSER, {} on {} images. Proceed?".format(SELECTED_VLAD, SELECTED_SUBSET)) print("Computing SIFT...") # 1. Compute SIFT descriptors from a dataset. The supported descriptors are ORB, SIFT and SURF: part = 'oxford5k_{}'.format(SELECTED_SUBSET) save_to = "pickles/{}_sifts_{}.h5".format(part, SELECTED_VLAD) patch_dict = {} if SELECTED_VLAD == 'pb-vlad': #print("images: ", data) sift_descriptors = load_h5(save_to, getPBDescriptors, data_dir, extractor(SELECTED_VLAD), optional_file_list=data) #sift_descriptors = load_h5(save_to, generate_patch_data, data_dir, data, patch_dict, rescale_images=False) #save_to = "pickles/{}_patchdict_{}.h5".format(part, SELECTED_VLAD) #patch_dict = load_joblib(save_to, lambda data: data, patch_dict) else: sift_descriptors = load_h5(save_to, getDescriptors, data_dir, extractor(SELECTED_VLAD), optional_file_list=data) print("SIFT shape: ", sift_descriptors.shape) print("Constructing VISUAL dictionary...") # 2. Construct a visual dictionary from the descriptors in path -d, with -w visual words: number_of_visual_words = 128 #visual_dictionary=kMeansDictionary(sift_descriptors, number_of_visual_words) #save_to = "pickles/vlad_dict_{}_{}.pickle".format(SELECTED_VLAD, part) #visual_dictionary = load_pickle(save_to, kMeansDictionary, sift_descriptors, number_of_visual_words) save_to = "pickles/{}_visual-dict_{}.joblib".format(part, 'orig-vlad') #, "orig-vlad") # test if orig-vlad dict helps: SELECTED_VLAD) visual_dictionary = load_joblib(save_to, kMeansDictionary, sift_descriptors, number_of_visual_words)
def remove_checksum(nrheaders): # nrheaders = 1 read_dir = '/home/mclrn/Data/linux/' headers = '{0}/'.format(nrheaders) dataframes = [] for fullname in glob.iglob(read_dir + headers + '*.h5'): filename = os.path.basename(fullname) df = utils.load_h5(read_dir + headers, filename) dataframes.append(df) # create one large dataframe df = pd.concat(dataframes) # df = pd.read_hdf('/home/mclrn/Data/salik_windows/{0}/'.format(nrheaders), key="extracted_{0}".format(nrheaders)) df = df.sample(frac=1).reset_index(drop=True) values, counts = np.unique(df['label'], return_counts=True) print(values, counts) #selector = df['label'] == 'youtube' values = df['bytes'].values bytes = np.zeros((values.shape[0], nrheaders * 54)) for i, v in enumerate(values): payload = np.zeros(nrheaders * 54, dtype=np.uint8) payload[:v.shape[0]] = v bytes[i] = payload #mean = np.mean(bytes, axis=0) #min = np.min(bytes, axis=0) #max = np.max(bytes, axis=0) #print(np.max(bytes[0:, 23])) # Protocol field if value = 6 then TCP if value = 17 the UDP bytes_no_checksum = [] for j, b in enumerate(bytes): if b[23] == 6: # TCP # if bytenumber in (50, 51): # return "Checksum (TCP header)" for i in range(nrheaders): b[i * 54 + 50] = 0 b[i * 54 + 51] = 0 b[i * 54 + 24] = 0 b[i * 54 + 25] = 0 elif b[23] == 17: # UDP # if bytenumber in (40,41) # return "UDP Checksum (UDP Header)" for i in range(nrheaders): b[i * 42 + 40] = 0 b[i * 42 + 41] = 0 b[i * 42 + 24] = 0 b[i * 42 + 25] = 0 else: print("Byte was not 6 nor 17 but: %d" % bytes[23]) bytes_no_checksum.append(b) new_data = {'bytes': bytes_no_checksum, 'label': df['label'].values} new_df = pd.DataFrame(new_data) # print(df) # print(new_df) save_dir = read_dir + "no_checksum/" + headers # if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) new_df.to_hdf(save_dir + "extracted_{0}-no_checksum".format(nrheaders) + '.h5', key='extracted_{0}'.format(nrheaders), mode='w')
def load(fname, fdir, fs, trial_start_end_seconds, conditions, num_avg_groups): """ Takes in a numpy 2d array and a subplot location, and plots a heatmap at the subplot location without axes Parameters ---------- fname : string file name fdir : string root file directory. Needs to have a "_framenumberforevents.pkl" file that corresponds to the session!! fs : float Sampling rate of the recording trial_start_end_seconds : list list with two float entries. First conditions : list list of strings that correspond to the behavioral conditions to be analyzed num_avg_groups : int Number of segments to split and average the trials over. Ie. Because single trial plots in state space is noisy, we break the trials up into groups and average to get less noisier signal. Returns ------- data_dict : dictionary 1st level of dict keys: individual conditions + condition combined data 2nd level of keys : data : numpy 4d array with dimensions (trials,y,x,samples) num_samples : number of samples (time) in a trial num_trials : total number of trials in the condition """ sima_h5_path = os.path.join(fdir, fname + '_sima_mc.h5') data_snip = utils.load_h5(sima_h5_path) data_dims = data_snip.shape tvec = np.linspace(0, data_dims[2] / fs, data_dims[2]) #load behavioral data and trial info try: glob_frame_files = glob.glob( fdir + "framenumberforevents*") # look for a file in specified directory frame_events = pickle.load( open(glob_frame_files[0], "rb"), encoding="latin1") # latin1 b/c original pickle made in python 2 except: print( 'Cannot find behavioral data file or file path is incorrect; utils.extract_trial_data will throw error.' ) # with trial start/end samples, trial_window_samp = trial_start_end_seconds * fs # turn trial start/end times to samples data_dict = utils.extract_trial_data(data_snip, trial_window_samp[0], trial_window_samp[1], frame_events, conditions) """let's load data into xarray format, which has numerous advantages over using numpy arrays, one of which is the ability to assign names to dimensions rather than indexing by ints """ for condition in conditions: # create index vectors for data dimensions; xarrayy stores these indices (eg. encodes time in seconds in place of samples) ypix_vec = range(0, data_dims[0]) xpix_vec = range(0, data_dims[1]) flattenpix_vec = range(0, data_dims[0] * data_dims[1]) trials_vec = range(data_dict[condition]['num_trials']) data_dict['trial_tvec'] = np.linspace( trial_start_end_seconds[0], trial_start_end_seconds[1], data_dict[condition]['num_samples']) # xarray with dimensions: x,y,trial,samples data_dict[condition]['xarr_data'] = xarray.DataArray( data_dict[condition]['data'], coords=[trials_vec, ypix_vec, xpix_vec, data_dict['trial_tvec']], dims=['trial', 'y', 'x', 'time']) # flatten x and y pixels into one dimension # reshape data and make xarray with dims: x-y,trial,samples flatten_pix_trial_data = np.reshape( data_dict[condition]['data'], (len(trials_vec), data_dims[0] * data_dims[1], len(data_dict['trial_tvec']))) data_dict[condition]['xarr_flatten_xy'] = xarray.DataArray( flatten_pix_trial_data, # this flattens only the x,y dimensions coords=[trials_vec, flattenpix_vec, data_dict['trial_tvec']], dims=['trial', 'yx', 'time']) # average across trials data_dict[condition]['xarr_flatten_pix_trialAvg'] = data_dict[ condition]['xarr_flatten_xy'].mean(dim='trial') ### https://stackoverflow.com/questions/43015638/xarray-reshape-data-split-dimension # unstack trials into groups and average across trials (avged trials grouped by time) num_trials_to_avg = data_dict[condition]['num_trials'] / num_avg_groups # need to create a pandas multi-index to tell xarray the target dimensions to unpack into ind = pd.MultiIndex.from_product( [np.arange(0, num_trials_to_avg), np.arange(0, num_avg_groups)], names=['trials', 'trial_groups' ])[np.arange(0, data_dict[condition]['num_trials'])] # last arange cuts the index list if the number of trials per group does divide evenly into total num trials data_dict[condition]['xarr_flatten_xy_group_trials'] = data_dict[ condition]['xarr_flatten_xy'].assign_coords( trial=ind).unstack('trial').mean(dim='trials').transpose( 'trial_groups', 'yx', 'time') ### # pull out all trial-avged data for each cond, then average across conditions data_dict['all_cond'] = {} # make an array with dimensions trials, xy_pixels, samples where trials from all conditions are stacked in the first dimension stacked_data = np.stack([ data_dict[condition]['xarr_flatten_xy'].data for condition in conditions ], axis=0) data_shape = stacked_data.shape data_dict['all_cond']['flattenpix'] = stacked_data.reshape( data_shape[0] * data_shape[1], data_shape[2], data_shape[3]) data_dict['all_cond']['flattenpix_trial_cond_avg'] = np.average([ data_dict[condition]['xarr_flatten_pix_trialAvg'].data for condition in conditions ], axis=0) return data_dict
def make_emb_db(self, args, net, data_loader, eval_sampled, eval_per_class, newly_trained=True, batch_size=None, mode='val'): """ :param batch_size: :param eval_sampled: :param eval_per_class: :param newly_trained: :param mode: :param args: utils args :param net: trained top_model network :param data_loader: DataLoader object :return: None """ if newly_trained: net.eval() if batch_size is None: batch_size = args.batch_size steps = int(np.ceil(len(data_loader) / batch_size)) test_classes = np.zeros(((len(data_loader.dataset)))) test_seen = np.zeros(((len(data_loader.dataset)))) test_paths = np.empty(dtype='S20', shape=((len(data_loader.dataset)))) if args.feat_extractor == 'resnet50': test_feats = np.zeros((len(data_loader.dataset), 2048)) elif args.feat_extractor == 'resnet18': test_feats = np.zeros((len(data_loader.dataset), 512)) else: raise Exception('Not handled feature extractor') for idx, (img, lbl, seen, path) in enumerate(data_loader): if args.cuda: img = img.cuda() img = Variable(img) output = net.forward(img, None, single=True) output = output.data.cpu().numpy() end = min((idx + 1) * batch_size, len(test_feats)) test_feats[idx * batch_size:end, :] = output test_classes[idx * batch_size:end] = lbl test_paths[idx * batch_size:end] = path test_seen[idx * batch_size:end] = seen.to(int) utils.save_h5(f'{mode}_ids', test_paths, 'S20', os.path.join(self.save_path, f'{mode}Ids.h5')) utils.save_h5(f'{mode}_classes', test_classes, 'i8', os.path.join(self.save_path, f'{mode}Classes.h5')) utils.save_h5(f'{mode}_feats', test_feats, 'f', os.path.join(self.save_path, f'{mode}Feats.h5')) utils.save_h5(f'{mode}_seen', test_seen, 'i2', os.path.join(self.save_path, f'{mode}Seen.h5')) test_feats = utils.load_h5( f'{mode}_feats', os.path.join(self.save_path, f'{mode}Feats.h5')) test_classes = utils.load_h5( f'{mode}_classes', os.path.join(self.save_path, f'{mode}Classes.h5')) test_seen = utils.load_h5( f'{mode}_seen', os.path.join(self.save_path, f'{mode}Seen.h5')) utils.calculate_k_at_n(args, test_feats, test_classes, test_seen, logger=self.logger, limit=args.limit_samples, run_number=args.number_of_runs, save_path=self.save_path, sampled=eval_sampled, per_class=eval_per_class, mode=mode) self.logger.info('results at: ' + self.save_path)
def compare_data(path1, path2, nrheaders): df1 = pd.DataFrame() df2 = pd.DataFrame() for fullname in glob.iglob(path1 + '*.h5'): filename = os.path.basename(fullname) df1 = utils.load_h5(path1, filename) for fullname in glob.iglob(path2 + '*.h5'): filename = os.path.basename(fullname) df2 = utils.load_h5(path2, filename) classes = [] # find all classes for label in set(df1['label']): classes.append(label) print(set(df1['label'])) print(set(df2['label'])) # filter on classes for c in classes: ## Exclude youtube as it contains both UDP and TCP if (c == 'youtube'): continue # create selector df1_selector = df1['label'] == c df2_selector = df2['label'] == c df1_values = df1[df1_selector]['bytes'].values df2_values = df2[df2_selector]['bytes'].values df1_bytes = np.zeros((df1_values.shape[0], nrheaders * 54)) df2_bytes = np.zeros((df2_values.shape[0], nrheaders * 54)) for i, v in enumerate(df1_values): payload = np.zeros(nrheaders * 54, dtype=np.uint8) payload[:v.shape[0]] = v df1_bytes[i] = payload for i, v in enumerate(df2_values): payload = np.zeros(nrheaders * 54, dtype=np.uint8) payload[:v.shape[0]] = v df2_bytes[i] = payload # Extract byte 23 to determine the protocol. TCP = True if int(df2_bytes[0][23]) == 6 else False df1_mean = np.mean(df1_bytes, axis=0) df2_mean = np.mean(df2_bytes, axis=0) df1_min = np.min(df1_bytes, axis=0) df2_min = np.min(df2_bytes, axis=0) df1_max = np.max(df1_bytes, axis=0) df2_max = np.max(df2_bytes, axis=0) for index, mean in enumerate(df1_mean): if (index % 25 == 0): print(c, index) if df1_mean[index] > 0 or df2_mean[index] > 0: if (int(df1_min[index]) != int(df1_max[index]) or int(df2_min[index]) != int(df2_max[index])): if (int(df1_mean[index]) != int(df2_mean[index]) or int(df1_min[index]) != int(df2_min[index]) or int(df1_max[index]) != int(df2_max[index])): print(index, " : ", int(df1_mean[index]), ' : ', int(df2_mean[index]), int(df1_min[index]), int(df2_min[index]), int(df1_max[index]), int(df2_max[index])) headername = byteindextoheaderfield(index, TCP) headername = headername.replace('/', ' ') createBoxplotsFromColumns( c + headername + ':' + str(index), df1_bytes[:, index], df2_bytes[:, index])
import pca.dataanalyzer import utils import glob import os import pandas as pd import numpy as np from matplotlib import pyplot as plt from pca.dataanalyzer import byteindextoheaderfield num_headers = 8 train_dir = 'C:/Users/salik/Documents/Data/LinuxChrome/{}/'.format(num_headers) dataframes = [] for fullname in glob.iglob(train_dir + '*.h5'): filename = os.path.basename(fullname) df = utils.load_h5(train_dir, filename) dataframes.append(df) # create one large dataframe data = pd.concat(dataframes) print(len(data)) print("drtv:", len(data[data['label'] == 'drtv'])) print("hbo:", len(data[data['label'] == 'hbo'])) print("http:", len(data[data['label'] == 'http'])) print("https:", len(data[data['label'] == 'https'])) print("netflix:", len(data[data['label'] == 'netflix'])) print("twitch:", len(data[data['label'] == 'twitch'])) print("youtube:", len(data[data['label'] == 'youtube'])) # dr_mean, dr_mean_sub, dr_std = getmeanstd(data, 'drtv') # nf_mean, nf_mean_sub, nf_std = getmeanstd(data, 'netflix') # mean_diff = dr_mean - nf_mean # sort_diff = (-abs(mean_diff)).argsort() #Sort on absolute values in decending order
def read_data_sets(train_dirs=[], test_dirs=None, merge_data=True, one_hot=False, dtype=dtypes.float32, validation_size=0.2, test_size=0.2, seed=None, balance_classes=False, payload_length=810): trainframes = [] testframes = [] for train_dir in train_dirs: for fullname in glob.iglob(train_dir + '*.h5'): filename = os.path.basename(fullname) df = utils.load_h5(train_dir, filename) trainframes.append(df) # create one large dataframe train_data = pd.concat(trainframes) if test_dirs != train_dirs: for test_dir in test_dirs: for fullname in glob.iglob(test_dir + '*.h5'): filename = os.path.basename(fullname) df = utils.load_h5(test_dir, filename) testframes.append(df) test_data = pd.concat(testframes) else: test_data = pd.DataFrame() if merge_data: train_data = pd.concat([test_data, train_data]) num_classes = len(train_data['label'].unique()) if balance_classes: values, counts = np.unique(train_data['label'], return_counts=True) smallest_class = np.argmin(counts) amount = counts[smallest_class] new_data = [] for v in values: sample = train_data.loc[train_data['label'] == v].sample(n=amount) new_data.append(sample) train_data = new_data train_data = pd.concat(train_data) # shuffle the dataframe and reset the index train_data = train_data.sample(frac=1, random_state=seed).reset_index(drop=True) # # youtube_selector = train_data['label'] == 'youtube' # youtube_data = train_data[youtube_selector] # for index, row in youtube_data.iterrows(): # bytes = row[0] # if bytes[23] == 17.0: # train_data.loc[index, 'label'] = 'youtube_udp' # else: # train_data.loc[index, 'label'] = 'youtube_tcp' if test_dirs != train_dirs: test_data = test_data.sample(frac=1, random_state=seed).reset_index(drop=True) test_labels = extract_labels(test_data, one_hot=one_hot, num_classes=num_classes) test_payloads = test_data['bytes'].values test_payloads = utils.pad_arrays_with_zero( test_payloads, payload_length=payload_length) train_labels = extract_labels(train_data, one_hot=one_hot, num_classes=num_classes) train_payloads = train_data['bytes'].values # pad with zero up to payload_length length train_payloads = utils.pad_arrays_with_zero(train_payloads, payload_length=payload_length) # TODO make seperate TEST SET ONCE ready total_length = len(train_payloads) validation_amount = int(total_length * validation_size) if merge_data: test_amount = int(total_length * test_size) test_payloads = train_payloads[:test_amount] test_labels = train_labels[:test_amount] val_payloads = train_payloads[test_amount:(validation_amount + test_amount)] val_labels = train_labels[test_amount:(validation_amount + test_amount)] train_payloads = train_payloads[(validation_amount + test_amount):] train_labels = train_labels[(validation_amount + test_amount):] else: val_payloads = train_payloads[:validation_amount] val_labels = train_labels[:validation_amount] train_payloads = train_payloads[validation_amount:] train_labels = train_labels[validation_amount:] options = dict(dtype=dtype, seed=seed) print("Training set size: {0}".format(len(train_payloads))) print("Validation set size: {0}".format(len(val_payloads))) print("Test set size: {0}".format(len(test_payloads))) train = DataSet(train_payloads, train_labels, **options) validation = DataSet(val_payloads, val_labels, **options) test = DataSet(test_payloads, test_labels, **options) return base.Datasets(train=train, validation=validation, test=test)