def read_data(): file_name = "train_data.dat" blob = np.fromfile(file_name, np.uint8) # reshape to images n_channels = 3 height = 32 width = 32 n_images = len(blob) // (n_channels * height * width) data = blob.reshape(n_images, height, width, n_channels) # resize resize_len = 224 images = np.empty((n_images, resize_len, resize_len, n_channels), np.uint8) for i in prange(n_images): images[i] = cv2.resize(data[i], (resize_len, resize_len)) # convert from [0,255] to [0.0,1.0] # normalize u2f_ratio = np.float32(255.0) c0_m = np.float32(0.485) c1_m = np.float32(0.456) c2_m = np.float32(0.406) c0_std = np.float32(0.229) c1_std = np.float32(0.224) c2_std = np.float32(0.225) for i in prange(n_images): images[i, :, :, 0] = (images[i, :, :, 0] / u2f_ratio - c0_m) / c0_std images[i, :, :, 1] = (images[i, :, :, 1] / u2f_ratio - c1_m) / c1_std images[i, :, :, 2] = (images[i, :, :, 2] / u2f_ratio - c2_m) / c2_std # convert to CHW images = images.transpose(0, 3, 1, 2) return images
def get_stats(imgs, num_detections, detection_classes, detection_scores): n_imgs = len(num_detections) car_mask = np.empty(n_imgs, np.bool_) top_detect_scores = detection_scores[:,0] num_cars = np.empty(n_imgs, np.int32) num_bikes = np.empty(n_imgs, np.int32) for i in prange(n_imgs): car_mask[i] = False num_cars[i] = 0 num_bikes[i] = 0 classes = detection_classes[i,:] for j in range(len(classes)): if classes[j] == CAR_CLASS: num_cars[i] += 1 car_mask[i] = True if classes[j] == BIKE_CLASS: num_bikes[i] += 1 df = pd.DataFrame({'top_detect_scores': top_detect_scores, 'num_detections': num_detections, 'num_cars': num_cars, 'num_bikes': num_bikes}) stats = df.describe() print(stats) car_imgs = imgs[car_mask] car_imgs.tofile("/export/intel/users/etotoni/cars.dat")
def read_data(): #fname = "/export/intel/lustre/etotoni/BXP5401-front-camera_2017.dat" fname = "img2.dat" blob = np.fromfile(fname, np.uint8) # reshape to images n_channels = 3 height = 800 width = 1280 n_images = len(blob) // (n_channels * height * width) data = blob.reshape(n_images, height, width, n_channels) # select every 5 image data = data[::5, :, :, :] n_images = len(data) # crop to 600 by 600 data = data[:, 100:-100, 340:-340, :] # resize resize_len = 224 resized_images = np.empty((n_images, resize_len, resize_len, n_channels), np.uint8) for i in prange(n_images): resized_images[i] = cv2.resize(data[i], (resize_len, resize_len)) # convert from [0,255] to [0.0,1.0] fdata = (resized_images) / np.float32(255.0) fdata = (fdata - np.float32(0.5)) / np.float32(0.5) # convert to CHW fdata = fdata.transpose((0, 3, 1, 2)) return fdata
def intraday_mean_revert(): file_name = "stock_data_all_google.hdf5" f = h5py.File(file_name, "r") sym_list = list(f.keys()) nsyms = len(sym_list) max_num_days = 4000 all_res = np.zeros(max_num_days) t1 = time.time() for i in prange(nsyms): symbol = sym_list[i] s_open = f[symbol + '/Open'][:] s_high = f[symbol + '/High'][:] s_low = f[symbol + '/Low'][:] s_close = f[symbol + '/Close'][:] s_vol = f[symbol + '/Volume'][:] df = pd.DataFrame({ 'Open': s_open, 'High': s_high, 'Low': s_low, 'Close': s_close, 'Volume': s_vol, }) # create column to hold our 90 day rolling standard deviation df['Stdev'] = df['Close'].rolling(window=90).std() # create a column to hold our 20 day moving average df['Moving Average'] = df['Close'].rolling(window=20).mean() # create a column which holds a TRUE value if the gap down from previous day's low to next # day's open is larger than the 90 day rolling standard deviation df['Criteria1'] = (df['Open'] - df['Low'].shift(1)) < -df['Stdev'] # create a column which holds a TRUE value if the opening price of the stock is above the 20 day moving average df['Criteria2'] = df['Open'] > df['Moving Average'] # create a column that holds a TRUE value if both above criteria are also TRUE df['BUY'] = df['Criteria1'] & df['Criteria2'] # calculate daily % return series for stock df['Pct Change'] = (df['Close'] - df['Open']) / df['Open'] # create a strategy return series by using the daily stock returns where the trade criteria above are met df['Rets'] = df['Pct Change'][df['BUY']] n_days = len(df['Rets']) res = np.zeros(max_num_days) if n_days: res[-n_days:] = df['Rets'].fillna(0).values all_res += res f.close() print(all_res.mean()) print("execution time:", time.time() - t1)
def read_example(): t1 = time.time() A = hpat.ros.read_ros_images("image_test.bag") # crop out dashboard B = A[:, :-50, :, :] # intensity threshold threshold = B.mean() + .004 * B.std() n = B.shape[0] mask = np.empty(n, np.bool_) for i in prange(n): im = B[i] mask[i] = im.mean() > threshold C = B[mask] D = np.empty_like(C) for i in prange(len(C)): D[i, :, :, 0] = gaussian_blur(C[i, :, :, 0]) D[i, :, :, 1] = gaussian_blur(C[i, :, :, 1]) D[i, :, :, 2] = gaussian_blur(C[i, :, :, 2]) # K-means model numCenter = 4 numIter = 10 dn, dh, dw, dc = D.shape centroids = np.random.randint(0, 255, (numCenter, dh, dw, dc)).astype(np.uint8) for l in range(numIter): dist = np.array([[ sqrt(np.sum((D[i] - centroids[j])**2)) for j in range(numCenter) ] for i in range(dn)]) labels = np.array([dist[i].argmin() for i in range(dn)]) for i in range(numCenter): mask2 = (labels == i) num_points = np.sum(mask2) if num_points != 0: centroids[i] = np.sum(D[mask2], 0) / num_points else: centroids[i] = np.random.randint(0, 255, (dh, dw, dc)).astype(np.uint8) t2 = time.time() print("Exec time: ", t2 - t1) return centroids
def test_impl(n): X = np.ones(n) b = 0.5 points = np.array([-1.0, 2.0, 5.0]) N = points.shape[0] exps = 0 for i in hpat.prange(n): p = X[i] d = (-(p - points)**2) / (2 * b**2) m = np.min(d) exps += m - np.log(b * N) + np.log(np.sum(np.exp(d - m))) return exps
def f(N, D, M): X = np.random.randint(0, 5, size=(N, D)).astype(np.int32) y = np.empty(N, dtype=np.int32) for i in prange(N): y[i] = i % 4 p = np.random.randint(0, 5, size=(M, D)).astype(np.int32) clf = hpat.ml.MultinomialNB(n_classes=4) t1 = time.time() clf.train(X, y) res = clf.predict(p) print("Exec time:", time.time() - t1) return res.sum()
def f(N, D, M): X = np.random.ranf((N, D)) y = np.empty(N) for i in prange(N): y[i] = i%4 p = np.random.ranf((M, D)) clf = hpat.ml.SVC(n_classes=4) t1 = time.time() clf.train(X, y) res = clf.predict(p) print("Exec time:", time.time()-t1) return res.sum()
def load_images(): files = glob.glob( "/home/etotoni/pse-hpc/python/datasets/KITTI/training/image_2/007*png") n = len(files) dataset = np.empty((n, 360, 1000, 3), np.uint8) t1 = time.time() for i in prange(n): f = files[i] img = cv2.imread(f) img_resized = cv2.resize(img, (1000, 360)) dataset[i] = img_resized m = dataset.mean() print("mean ", m, "\nExec time", time.time() - t1)
def kde(): f = h5py.File("kde.hdf5", "r") X = f['points'][:] b = 0.5 points = np.array([-1.0, 2.0, 5.0]) N = points.shape[0] n = X.shape[0] exps = 0 t1 = time.time() for i in prange(n): p = X[i] d = (-(p - points)**2) / (2 * b**2) m = np.min(d) exps += m - np.log(b * N) + np.log(np.sum(np.exp(d - m))) t = time.time() - t1 print("Execution time:", t, "\nresult:", exps) return exps
def intraday_mean_revert(): nsyms = 500 max_num_days = 4000 all_res = np.zeros(max_num_days) t1 = time.time() for i in prange(nsyms): s_open = 20 * np.random.randn(max_num_days) s_high = 22 * np.random.randn(max_num_days) s_low = 18 * np.random.randn(max_num_days) s_close = 19 * np.random.randn(max_num_days) s_vol = 1000 * np.random.randn(max_num_days) df = pd.DataFrame({ 'Open': s_open, 'High': s_high, 'Low': s_low, 'Close': s_close, 'Volume': s_vol, }) #create column to hold our 90 day rolling standard deviation df['Stdev'] = df['Close'].rolling(window=90).std() #create a column to hold our 20 day moving average df['Moving Average'] = df['Close'].rolling(window=20).mean() #create a column which holds a TRUE value if the gap down from previous day's low to next #day's open is larger than the 90 day rolling standard deviation df['Criteria1'] = (df['Open'] - df['Low'].shift(1)) < -df['Stdev'] #create a column which holds a TRUE value if the opening price of the stock is above the 20 day moving average df['Criteria2'] = df['Open'] > df['Moving Average'] #create a column that holds a TRUE value if both above criteria are also TRUE df['BUY'] = df['Criteria1'] & df['Criteria2'] #calculate daily % return series for stock df['Pct Change'] = (df['Close'] - df['Open']) / df['Open'] #create a strategy return series by using the daily stock returns where the trade criteria above are met df['Rets'] = df['Pct Change'][df['BUY'] == True] all_res += df['Rets'].fillna(0) print(all_res.mean()) print("execution time:", time.time() - t1)
def kde(): t = pq.read_table('kde.parquet') df = t.to_pandas() X = df['points'].values b = 0.5 points = np.array([-1.0, 2.0, 5.0]) N = points.shape[0] n = X.shape[0] exps = 0 t1 = time.time() for i in prange(n): p = X[i] d = (-(p - points)**2) / (2 * b**2) m = np.min(d) exps += m - np.log(b * N) + np.log(np.sum(np.exp(d - m))) t = time.time() - t1 print("Execution time:", t, "\nresult:", exps) return exps
def test_impl(nsyms): max_num_days = 100 all_res = 0.0 for i in hpat.prange(nsyms): s_open = 20 * np.ones(max_num_days) s_low = 28 * np.ones(max_num_days) s_close = 19 * np.ones(max_num_days) df = pd.DataFrame({'Open': s_open, 'Low': s_low, 'Close': s_close}) df['Stdev'] = df['Close'].rolling(window=90).std() df['Moving Average'] = df['Close'].rolling(window=20).mean() df['Criteria1'] = (df['Open'] - df['Low'].shift(1)) < -df['Stdev'] df['Criteria2'] = df['Open'] > df['Moving Average'] df['BUY'] = df['Criteria1'] & df['Criteria2'] df['Pct Change'] = (df['Close'] - df['Open']) / df['Open'] df['Rets'] = df['Pct Change'][df['BUY'] == True] all_res += df['Rets'].mean() return all_res