def handler(event, context): """ entry point for Lambda function :param event: the Lambda event :param context: the Lambda context :return: None """ print(f"'event': {event}") print(f"'context': {context}") # ----------------------------------------------------- # EXTRACT # define ny_dataset ny_dataset = classes.Dataset("ny_dataset") ny_dataset.headers_all = ["date", "cases", "deaths"] ny_dataset.headers_key = ny_dataset.headers_all ny_dataset.match_field = "date" ny_dataset.source_url = "https://raw.githubusercontent.com/nytimes/covid-19-data/master/us.csv" # extract and print ny_dataset ny_dataset.df = extract.extract(ny_dataset.source_url) print(f"'ny_dataset.df':\n{ny_dataset.df}") # define jh_dataset jh_dataset = classes.Dataset("jh_dataset") jh_dataset.headers_all = [ "Date", "Country/Region", "Province/State", "Lat", "Long", "Confirmed", "Recovered", "Deaths" ] jh_dataset.headers_key = ["Date", "Country/Region", "Recovered"] jh_dataset.match_field = "Date" jh_dataset.source_url = \ "https://raw.githubusercontent.com/datasets/covid-19/master/data/time-series-19-covid-combined.csv" # extract and print jh_dataset jh_dataset.df = extract.extract(jh_dataset.source_url, jh_dataset.headers_key, "Country/Region", "US") print(f"'jh_dataset.df':\n{jh_dataset.df}") # ----------------------------------------------------- # TRANSFORM # transform the datasets into CovidStat Instances covid_stats = transform.transform(ny_dataset, jh_dataset) # print CovidStats print(*covid_stats, sep="\n") # ----------------------------------------------------- # LOAD # load CovidStat instances into the CovidStats DynamoDB table load.load_all(classes.CovidStat, covid_stats) load.load_json(covid_stats)
def main(trace_directory, trace_name, number): trace_directory = abspath(trace_directory) ptfile = '%s/pt_%s.txt' % (trace_directory, trace_name) pt_data = load_all(ptfile, number=number)[:-1] textin = np.array(pt_data) traces = load_traces('%s/avg_%s' % (trace_directory, trace_name), number) assert traces.shape[0] == textin.shape[0] with open('tracefile', 'wb') as f: for t in traces.flatten(): f.write(struct.pack('f', t)) with open('plaintext', 'wb') as f: for t in textin.flatten(): f.write(struct.pack('B', t)) print("Add following to CONFIG:\n") print("[Traces]") print("files=1") print("trace_type=f") print("transpose=true") print("index=0") print("nsamples=%d" % traces.shape[1]) print("trace=tracefile %d %d" % (traces.shape[0], traces.shape[1])) print("") print("[Guesses]") print("files=1") print("guess_type=u") print("transpose=true") print("guess=plaintext %d %d" % (textin.shape[0], textin.shape[1]))
def feature_aegan(aegan, modelname, protoname): with ipdb.launch_ipdb_on_exception(): aegan.load(prefix=modelname) x = transform(load_all(protoname, (npxw, npxh))) code = aegan.autoencoder.encoder.predict(x) ipdb.set_trace()
def test_load(self): """ :return: pass or fail if the ... """ print("test_load") size = 25 sample_date = list() for _ in range(size): sample_date.append(create_random_instance()) load_response = load.load_all(CovidStatTest, sample_date) self.assertEqual(len(load_response), size)
def cli(data_path, num_traces, start_point, end_point, plot, num_key_bytes, bruteforce, name): """ Run an attack against previously collected traces. Each attack is a separate subcommand. The options to the top-level command apply to all attacks; see the individual attacks' documentation for attack-specific options. """ global PLOT, NUM_KEY_BYTES, BRUTEFORCE, PLAINTEXTS, TRACES, KEYFILE PLOT = plot NUM_KEY_BYTES = num_key_bytes BRUTEFORCE = bruteforce PLAINTEXTS = load_all(path.join(data_path, 'pt_%s.txt' % name)) TRACES = load_traces(path.join(data_path, 'avg_%s' % name), num_traces or len(PLAINTEXTS), start_point, end_point) KEYFILE = path.join(data_path, 'key_%s.txt' % name)
for index, time in enumerate(times[1:], start=1): diff = time - times[index - 1] # 30 minutes for one view if diff > 30 * 60 * 1000 or index == len(times) - 1: blockTimes.append(times[index - 1] - blockBegin) spentTime += times[index - 1] - blockBegin blockBegin = time blocks += 1 blockStop.append(index - 1) print(f'{name}: {len(times)} entries, {blocks} blocks') print(f'Block avg: {displayTime(np.average(blockTimes))}') print(f'Entry avg: {displayTime(np.average(spentTime//len(times)))}') print(f'Total: {displayTime(spentTime)}') print(f'Skips: {skip:>7}') return spentTime dataAll = load_all(args.rating_dir) total = 0 for (name, data) in dataAll.items(): total += time_single(name, data) print() print(f'Total:') print(f'Time: {displayTime(total)}') print(f'Entries: {totalEntries:>7}') print(f'Avg.: {displayTime(total/totalEntries)}') print(f'Model avg.: {displayTime(total/totalEntries/13)}') print(f'Faulty: {totalFaulty:>7}')
def tra_create(template_dir, num_pois, poi_spacing): """ Template Radio Analysis; create a template. The data set should have a considerable size in order to allow for the construction of an accurate model. In general, the more data is used for template creation the less is needed to apply the template. The template directory is where we store multiple files comprising the template; beware that existing files will be overwritten! """ try: os.makedirs(template_dir) except OSError: # Checking the directory before attempting to create it leads to race # conditions. if not path.isdir(template_dir): raise if PLOT: plt.plot(np.average(TRACES,axis=0),'b') plt.show() tempKey = load_all(KEYFILE) fixed_key = (np.shape(tempKey)[0] == 1) for knum in range(NUM_KEY_BYTES): if(fixed_key): tempSbox = [sbox[PLAINTEXTS[i][knum] ^ tempKey[0][knum]] for i in range(len(TRACES))] else: tempSbox = [sbox[PLAINTEXTS[i][knum] ^ tempKey[i][knum]] for i in range(len(TRACES))] tempHW = [hw[s] for s in tempSbox] # Sort traces by HW # Make 9 blank lists - one for each Hamming weight tempTracesHW = [[] for _ in range(9)] # Fill them up for i, trace in enumerate(TRACES): HW = tempHW[i] tempTracesHW[HW].append(trace) # Check to have at least a trace for each HW for HW in range(9): assert len(tempTracesHW[HW]) != 0, "No trace with HW = %d, try increasing the number of traces" % HW # Switch to numpy arrays tempTracesHW = [np.array(tempTracesHW[HW]) for HW in range(9)] # Find averages tempMeans = np.zeros((9, len(TRACES[0]))) for i in range(9): tempMeans[i] = np.average(tempTracesHW[i], 0) # Find sum of differences tempSumDiff = np.zeros(len(TRACES[0])) for i in range(9): for j in range(i): tempSumDiff += np.abs(tempMeans[i] - tempMeans[j]) if PLOT: plt.plot(tempSumDiff,label="subkey %d"%knum) plt.legend() # Find POIs POIs = [] for i in range(num_pois): # Find the max nextPOI = tempSumDiff.argmax() POIs.append(nextPOI) # Make sure we don't pick a nearby value poiMin = max(0, nextPOI - poi_spacing) poiMax = min(nextPOI + poi_spacing, len(tempSumDiff)) for j in range(poiMin, poiMax): tempSumDiff[j] = 0 # Fill up mean and covariance matrix for each HW meanMatrix = np.zeros((9, num_pois)) covMatrix = np.zeros((9, num_pois, num_pois)) for HW in range(9): for i in range(num_pois): # Fill in mean meanMatrix[HW][i] = tempMeans[HW][POIs[i]] for j in range(num_pois): x = tempTracesHW[HW][:,POIs[i]] y = tempTracesHW[HW][:,POIs[j]] covMatrix[HW,i,j] = cov(x, y) with open(path.join(template_dir, 'POIs_%d' % knum), 'wb') as fp: pickle.dump(POIs, fp) with open(path.join(template_dir, 'covMatrix_%d' % knum), 'wb') as fp: pickle.dump(covMatrix, fp) with open(path.join(template_dir, 'meanMatrix_%d' % knum), 'wb') as fp: pickle.dump(meanMatrix, fp) if PLOT: plt.show()
import os import sys import numpy as np import argparse from scipy import misc import load, proc, densecrf import classes as cl ar = argparse.ArgumentParser() ar.add_argument('path1', type=str, help="Path to label images") args = ar.parse_args() path1 = args.path1 path1 = path1 + '/' path2 = '../../data/real/' result = path1 + 'crf/' if not os.path.exists(result): os.makedirs(result) labels = load.load_data_simp([path1]) set_imgs = load.load_all(path2, comp=True) x_img = set_imgs[1] x_img = proc._join(x_img, set_imgs[2:]) for i in range(len(x_img)): x_i = x_img[i] label = labels[0][i] crflabel = densecrf.crf(label, x_i, classes=2, ls=[0, 3]) crflabel = cl.declassimg2(crflabel, map=True) number = str(i).zfill(4) misc.imsave(result + number + ".png", crflabel)
robot_data_list = list() for marker in robot_markers: marker_data = data_by_marker_name(marker, data, metadata) robot_data_list.append(marker_data) robot_data = np.stack(robot_data_list, axis=2) robot_data = np.nanmean(robot_data, axis=2) human_data = data_by_marker_name("UsersHead", data, metadata) distance = np.linalg.norm(robot_data - human_data, axis=1) return distance if __name__ == "__main__": by_id, by_position = load_all("../tsv_files/*.tsv") #plot comfort #""" def plot_comfort(trial): metadata = trial["metadata"] data = trial["data"] eps = 1e-8 comfort = get_comfort(data, metadata) plt.plot(comfort) user_data = by_id[4] for position in ["A", "B", "C", "D", "E"]: plot_comfort(user_data[position]) plt.show()
loc += 2 for p in range(ps[1],x_i.shape[1]//2,ps[1]):#vertup result_img[0:ps[0],p:p+ps[1],:] = result[loc] result_img[-ps[0]:,p:p+ps[1],:] = result[loc+1] loc += 2 for p in range(x_i.shape[1]-ps[1],x_i.shape[1]//2,-ps[1]):#vertup result_img[0:ps[0],p-ps[1]:p,:] = result[loc] result_img[-ps[0]:,p-ps[1]:p,:] = result[loc+1] loc += 2 return result_img #----Load Data--- set_imgs = load.load_all(REAL_PATH,comp=comp) print("Loading done") print("Pre-processing") x_img = set_imgs[1] y_img = set_imgs[0] x_img = proc._join(x_img,set_imgs[2:]) _d = x_img[0].shape[2] #hnv_imgs = load.load_data_simp([REAL_PATH+'hnv_ng']) #x_img = proc._join(x_img,hnv_imgs[0:1],depth=2) model = load_model(TRAIN_PATH+'/model.hdf5',compile=False) print("Model loaded") _w = patch_size[0]
plt.savefig('test/{}.pdf'.format(ind)) plt.clf() ipdb.set_trace() if __name__ == '__main__': nbatch = 128 nmax = nbatch * 100 npxw, npxh = 64, 128 from load import people, load_all va_data, tr_stream, _ = people(pathfile='protocol/cuhk01-train.txt', size=(npxw, npxh), batch_size=nbatch) allx = transform(load_all('protocol/cuhk01-train.txt', (npxw, npxh))) g = Generator(g_size=(3, npxh, npxw), g_nb_filters=128, g_nb_coding=5000, g_scales=4, g_init=InitNormal(scale=0.002)) #, g_FC=[5000]) d = Discriminator(d_size=g.g_size, d_nb_filters=128, d_scales=4, d_init=InitNormal(scale=0.002)) #, d_FC=[5000]) # init with autoencoder ae = Autoencoder(g, d) # ae.fit(tr_stream, # save_dir='./samples/reid_aegan_5000/ae/',
#!/usr/bin/env python3 from load import load_all import numpy as np raise Exception("This file does not work with the new Pandas update.") data = load_all() modelCDTn = 'CUNI-DocTransformer' modelCTTn = 'CUNI-T2T-2018' modelREFn = 'ref' for (userKey, user) in data.items(): for (docKey, document) in user.items(): for (line, line) in document.items(): modelCDT = line[modelCDTn] modelCTT = line[modelCTTn] modelREF = line[modelREFn] if 'fluency' not in modelCDT or 'adequacy' not in modelCDT: continue if 'fluency' not in modelCTT or 'adequacy' not in modelCTT: continue if 'fluency' not in modelREF or 'adequacy' not in modelREF: continue scoreCDT = float(modelCDT['fluency']) * float(modelCDT['adequacy']) scoreCTT = float(modelCTT['fluency']) * float(modelCTT['adequacy']) scoreREF = float(modelREF['fluency']) * float(modelREF['adequacy']) if scoreCDT > scoreREF + 0.5 and scoreCDT > scoreCTT + 0.5:
def setup(device, name, interval, patch_size, comp, labels): TRAIN_PATH = OUT_PATH + name if not os.path.exists(TRAIN_PATH): print("Model does not exist") sys.exit(0) append = '_test' if not comp else '_test_comp' RESULT_PATH_BASE = TRAIN_PATH + append ''' if os.path.exists(RESULT_PATH_BASE): i = 0 while os.path.exists(RESULT_PATH_BASE): RESULT_PATH_BASE = TRAIN_PATH+append+'_'+str(i) i += 1 ''' if not os.path.exists(RESULT_PATH_BASE): os.makedirs(RESULT_PATH_BASE) RESULT_PATH_BASE = RESULT_PATH_BASE + '/' + str(interval) os.makedirs(RESULT_PATH_BASE) os.makedirs(RESULT_PATH_BASE + '/crf') os.makedirs(RESULT_PATH_BASE + '/avg') #----KERAS ENV------- os.environ["THEANO_FLAGS"] = 'device=' + device sys.setrecursionlimit(50000) import load, proc, classes, math from keras.models import Model, load_model #----Load Data--- set_imgs = load.load_all(REAL_PATH, comp=comp) print("Loading done") print("Pre-processing") x_img = set_imgs[1] y_img = set_imgs[0] x_img = proc._join(x_img, set_imgs[2:]) _d = x_img[0].shape[2] #hnv_imgs = load.load_data_simp([REAL_PATH+'hnv_ng']) #x_img = proc._join(x_img,hnv_imgs[0:1],depth=2) model = load_model(TRAIN_PATH + '/model.hdf5', compile=False) print("Model loaded") _w = patch_size[0] _h = patch_size[1] _w_out = _w out_depth = None ranges = range(-(patch_size[0] // 2), (patch_size[0] // 2), interval) #ranges = range(0,(patch_size[0]//2),interval) offsets = [] for xoff in ranges: for yoff in ranges: if yoff == xoff: offsets.append((xoff, yoff)) i = -1 '''for x_i in x_img:
def generate_dataset(): by_id, by_position = load_all("../tsv_files/*.tsv") positions = ["A", "B", "C", "D", "E"] dataset = np.empty((0, 10)) # magic number, cuz I'm a wizzard ya know? =) for user, data in enumerate(by_id): for position, trial in data.items(): metadata = trial["metadata"] raw_data = trial["data"] eps = 1e-8 #frame counter frames = raw_data[:, 0] - raw_data[0, 0] #robot position robot_markers = [ "DoubleBottom", "DoubleFaceBottomRight", "DoubleFaceTopRight", "DoubleFaceTopLeft", "DoubleFaceBottomLeft" ] marker_positions = list() for marker in robot_markers: try: marker_position = np.stack(marker_projection( marker, raw_data, metadata), axis=1) except TypeError: print(user, position, metadata) raise marker_positions.append(marker_position) marker_positions = np.stack(marker_positions, axis=2) robot_pos = np.nanmean(marker_positions, axis=2) #participant's comfort during the approach (lower is better) comfort = get_comfort(raw_data, metadata) #participant's height height = participant_height(raw_data, metadata) if not type(height) is np.ndarray: # height is 0.0 because head markers couldn't be found height = np.array([np.nan] * robot_pos.shape[0]) #human position human_pos = np.stack(marker_projection("UsersHead", raw_data, metadata), axis=1) # distance -- euclidean distance distance = get_distance(raw_data, metadata) #position and user label pos_label = np.array([positions.index(position)] * robot_pos.shape[0]) user_label = np.array([user] * robot_pos.shape[0]) #stack the data together data_rows = [ frames, robot_pos[:, 0], robot_pos[:, 1], distance, human_pos[:, 0], human_pos[:, 1], height, comfort, pos_label, user_label ] partial_dataset = np.stack(data_rows, axis=1) dataset = np.append(dataset, partial_dataset, axis=0) return dataset
train_val_splitting from define_metrics import Metrics_DDI_classification, multi_micro_f1_, \ get_callbacks_list from utils import argparser, combine_params from network_functions import define_network, train_network, get_predictions from calculate_metrics import get_metrics_cv if __name__ == '__main__': in_path, out_path, out_path_figure = argparser() #in_path = '/content/gdrive/My Drive/Master/Subjects/TFM/code/DOC/two_steps/second_step/' ## 1. Load embedding, position matrices and word2int_dict of train set, # train and test sets emb_matrix,pos_matrix,w2i_dict,VOCAB_SIZE,WV_VECTOR_SIZE,POS_VECTOR_LENGTH,\ MAX_SENTENCE_LENGTH,X_train,y_train,X_test,y_test = load_all(in_path) ## 2. Preprocessing label2int = {'INT': 0, 'ADVISE': 1, 'EFFECT': 2, 'MECHANISM': 3} X_train_i_pad, X_train_p1_pad, X_train_p2_pad, y_train_1h, y_train_i = \ prepro(X_train,y_train, w2i_dict, label2int, MAX_SENTENCE_LENGTH) X_test_i_pad, X_test_p1_pad, X_test_p2_pad, y_test_1h, y_test_i = \ prepro(X_test,y_test, w2i_dict, label2int, MAX_SENTENCE_LENGTH, test=True, y_trainset=y_train) original_train_data = y_train_i, y_train_1h, X_train_i_pad, X_train_p1_pad, X_train_p2_pad ## 3. Define metrics #out_path = "/content/gdrive/My Drive/Master/Subjects/TFM/code/DOC/two_steps/second_step/new/" metrics_callbacks = Metrics_DDI_classification() ## 4. Define network parameters