def read_data():
file_name = "train_data.dat"
blob = np.fromfile(file_name, np.uint8)
# reshape to images
n_channels = 3
height = 32
width = 32
n_images = len(blob) // (n_channels * height * width)
data = blob.reshape(n_images, height, width, n_channels)
# resize
resize_len = 224
images = np.empty((n_images, resize_len, resize_len, n_channels), np.uint8)
for i in prange(n_images):
images[i] = cv2.resize(data[i], (resize_len, resize_len))
# convert from [0,255] to [0.0,1.0]
# normalize
u2f_ratio = np.float32(255.0)
c0_m = np.float32(0.485)
c1_m = np.float32(0.456)
c2_m = np.float32(0.406)
c0_std = np.float32(0.229)
c1_std = np.float32(0.224)
c2_std = np.float32(0.225)
for i in prange(n_images):
images[i, :, :, 0] = (images[i, :, :, 0] / u2f_ratio - c0_m) / c0_std
images[i, :, :, 1] = (images[i, :, :, 1] / u2f_ratio - c1_m) / c1_std
images[i, :, :, 2] = (images[i, :, :, 2] / u2f_ratio - c2_m) / c2_std
# convert to CHW
images = images.transpose(0, 3, 1, 2)
return images
def get_stats(imgs, num_detections, detection_classes, detection_scores):
n_imgs = len(num_detections)
car_mask = np.empty(n_imgs, np.bool_)
top_detect_scores = detection_scores[:,0]
num_cars = np.empty(n_imgs, np.int32)
num_bikes = np.empty(n_imgs, np.int32)
for i in prange(n_imgs):
car_mask[i] = False
num_cars[i] = 0
num_bikes[i] = 0
classes = detection_classes[i,:]
for j in range(len(classes)):
if classes[j] == CAR_CLASS:
num_cars[i] += 1
car_mask[i] = True
if classes[j] == BIKE_CLASS:
num_bikes[i] += 1
df = pd.DataFrame({'top_detect_scores': top_detect_scores,
'num_detections': num_detections,
'num_cars': num_cars,
'num_bikes': num_bikes})
stats = df.describe()
print(stats)
car_imgs = imgs[car_mask]
car_imgs.tofile("/export/intel/users/etotoni/cars.dat")
def read_data():
#fname = "/export/intel/lustre/etotoni/BXP5401-front-camera_2017.dat"
fname = "img2.dat"
blob = np.fromfile(fname, np.uint8)
# reshape to images
n_channels = 3
height = 800
width = 1280
n_images = len(blob) // (n_channels * height * width)
data = blob.reshape(n_images, height, width, n_channels)
# select every 5 image
data = data[::5, :, :, :]
n_images = len(data)
# crop to 600 by 600
data = data[:, 100:-100, 340:-340, :]
# resize
resize_len = 224
resized_images = np.empty((n_images, resize_len, resize_len, n_channels),
np.uint8)
for i in prange(n_images):
resized_images[i] = cv2.resize(data[i], (resize_len, resize_len))
# convert from [0,255] to [0.0,1.0]
fdata = (resized_images) / np.float32(255.0)
fdata = (fdata - np.float32(0.5)) / np.float32(0.5)
# convert to CHW
fdata = fdata.transpose((0, 3, 1, 2))
return fdata
def intraday_mean_revert():
file_name = "stock_data_all_google.hdf5"
f = h5py.File(file_name, "r")
sym_list = list(f.keys())
nsyms = len(sym_list)
max_num_days = 4000
all_res = np.zeros(max_num_days)
t1 = time.time()
for i in prange(nsyms):
symbol = sym_list[i]
s_open = f[symbol + '/Open'][:]
s_high = f[symbol + '/High'][:]
s_low = f[symbol + '/Low'][:]
s_close = f[symbol + '/Close'][:]
s_vol = f[symbol + '/Volume'][:]
df = pd.DataFrame({
'Open': s_open,
'High': s_high,
'Low': s_low,
'Close': s_close,
'Volume': s_vol,
})
# create column to hold our 90 day rolling standard deviation
df['Stdev'] = df['Close'].rolling(window=90).std()
# create a column to hold our 20 day moving average
df['Moving Average'] = df['Close'].rolling(window=20).mean()
# create a column which holds a TRUE value if the gap down from previous day's low to next
# day's open is larger than the 90 day rolling standard deviation
df['Criteria1'] = (df['Open'] - df['Low'].shift(1)) < -df['Stdev']
# create a column which holds a TRUE value if the opening price of the stock is above the 20 day moving average
df['Criteria2'] = df['Open'] > df['Moving Average']
# create a column that holds a TRUE value if both above criteria are also TRUE
df['BUY'] = df['Criteria1'] & df['Criteria2']
# calculate daily % return series for stock
df['Pct Change'] = (df['Close'] - df['Open']) / df['Open']
# create a strategy return series by using the daily stock returns where the trade criteria above are met
df['Rets'] = df['Pct Change'][df['BUY']]
n_days = len(df['Rets'])
res = np.zeros(max_num_days)
if n_days:
res[-n_days:] = df['Rets'].fillna(0).values
all_res += res
f.close()
print(all_res.mean())
print("execution time:", time.time() - t1)
def read_example():
t1 = time.time()
A = hpat.ros.read_ros_images("image_test.bag")
# crop out dashboard
B = A[:, :-50, :, :]
# intensity threshold
threshold = B.mean() + .004 * B.std()
n = B.shape[0]
mask = np.empty(n, np.bool_)
for i in prange(n):
im = B[i]
mask[i] = im.mean() > threshold
C = B[mask]
D = np.empty_like(C)
for i in prange(len(C)):
D[i, :, :, 0] = gaussian_blur(C[i, :, :, 0])
D[i, :, :, 1] = gaussian_blur(C[i, :, :, 1])
D[i, :, :, 2] = gaussian_blur(C[i, :, :, 2])
# K-means model
numCenter = 4
numIter = 10
dn, dh, dw, dc = D.shape
centroids = np.random.randint(0, 255,
(numCenter, dh, dw, dc)).astype(np.uint8)
for l in range(numIter):
dist = np.array([[
sqrt(np.sum((D[i] - centroids[j])**2)) for j in range(numCenter)
] for i in range(dn)])
labels = np.array([dist[i].argmin() for i in range(dn)])
for i in range(numCenter):
mask2 = (labels == i)
num_points = np.sum(mask2)
if num_points != 0:
centroids[i] = np.sum(D[mask2], 0) / num_points
else:
centroids[i] = np.random.randint(0, 255,
(dh, dw, dc)).astype(np.uint8)
t2 = time.time()
print("Exec time: ", t2 - t1)
return centroids
def test_impl(n):
X = np.ones(n)
b = 0.5
points = np.array([-1.0, 2.0, 5.0])
N = points.shape[0]
exps = 0
for i in hpat.prange(n):
p = X[i]
d = (-(p - points)**2) / (2 * b**2)
m = np.min(d)
exps += m - np.log(b * N) + np.log(np.sum(np.exp(d - m)))
return exps
def f(N, D, M):
X = np.random.randint(0, 5, size=(N, D)).astype(np.int32)
y = np.empty(N, dtype=np.int32)
for i in prange(N):
y[i] = i % 4
p = np.random.randint(0, 5, size=(M, D)).astype(np.int32)
clf = hpat.ml.MultinomialNB(n_classes=4)
t1 = time.time()
clf.train(X, y)
res = clf.predict(p)
print("Exec time:", time.time() - t1)
return res.sum()
def f(N, D, M):
X = np.random.ranf((N, D))
y = np.empty(N)
for i in prange(N):
y[i] = i%4
p = np.random.ranf((M, D))
clf = hpat.ml.SVC(n_classes=4)
t1 = time.time()
clf.train(X, y)
res = clf.predict(p)
print("Exec time:", time.time()-t1)
return res.sum()
def load_images():
files = glob.glob(
"/home/etotoni/pse-hpc/python/datasets/KITTI/training/image_2/007*png")
n = len(files)
dataset = np.empty((n, 360, 1000, 3), np.uint8)
t1 = time.time()
for i in prange(n):
f = files[i]
img = cv2.imread(f)
img_resized = cv2.resize(img, (1000, 360))
dataset[i] = img_resized
m = dataset.mean()
print("mean ", m, "\nExec time",
time.time() - t1)
def kde():
f = h5py.File("kde.hdf5", "r")
X = f['points'][:]
b = 0.5
points = np.array([-1.0, 2.0, 5.0])
N = points.shape[0]
n = X.shape[0]
exps = 0
t1 = time.time()
for i in prange(n):
p = X[i]
d = (-(p - points)**2) / (2 * b**2)
m = np.min(d)
exps += m - np.log(b * N) + np.log(np.sum(np.exp(d - m)))
t = time.time() - t1
print("Execution time:", t, "\nresult:", exps)
return exps
def intraday_mean_revert():
nsyms = 500
max_num_days = 4000
all_res = np.zeros(max_num_days)
t1 = time.time()
for i in prange(nsyms):
s_open = 20 * np.random.randn(max_num_days)
s_high = 22 * np.random.randn(max_num_days)
s_low = 18 * np.random.randn(max_num_days)
s_close = 19 * np.random.randn(max_num_days)
s_vol = 1000 * np.random.randn(max_num_days)
df = pd.DataFrame({
'Open': s_open,
'High': s_high,
'Low': s_low,
'Close': s_close,
'Volume': s_vol,
})
#create column to hold our 90 day rolling standard deviation
df['Stdev'] = df['Close'].rolling(window=90).std()
#create a column to hold our 20 day moving average
df['Moving Average'] = df['Close'].rolling(window=20).mean()
#create a column which holds a TRUE value if the gap down from previous day's low to next
#day's open is larger than the 90 day rolling standard deviation
df['Criteria1'] = (df['Open'] - df['Low'].shift(1)) < -df['Stdev']
#create a column which holds a TRUE value if the opening price of the stock is above the 20 day moving average
df['Criteria2'] = df['Open'] > df['Moving Average']
#create a column that holds a TRUE value if both above criteria are also TRUE
df['BUY'] = df['Criteria1'] & df['Criteria2']
#calculate daily % return series for stock
df['Pct Change'] = (df['Close'] - df['Open']) / df['Open']
#create a strategy return series by using the daily stock returns where the trade criteria above are met
df['Rets'] = df['Pct Change'][df['BUY'] == True]
all_res += df['Rets'].fillna(0)
print(all_res.mean())
print("execution time:", time.time() - t1)
def kde():
t = pq.read_table('kde.parquet')
df = t.to_pandas()
X = df['points'].values
b = 0.5
points = np.array([-1.0, 2.0, 5.0])
N = points.shape[0]
n = X.shape[0]
exps = 0
t1 = time.time()
for i in prange(n):
p = X[i]
d = (-(p - points)**2) / (2 * b**2)
m = np.min(d)
exps += m - np.log(b * N) + np.log(np.sum(np.exp(d - m)))
t = time.time() - t1
print("Execution time:", t, "\nresult:", exps)
return exps
def test_impl(nsyms):
max_num_days = 100
all_res = 0.0
for i in hpat.prange(nsyms):
s_open = 20 * np.ones(max_num_days)
s_low = 28 * np.ones(max_num_days)
s_close = 19 * np.ones(max_num_days)
df = pd.DataFrame({'Open': s_open, 'Low': s_low,
'Close': s_close})
df['Stdev'] = df['Close'].rolling(window=90).std()
df['Moving Average'] = df['Close'].rolling(window=20).mean()
df['Criteria1'] = (df['Open'] - df['Low'].shift(1)) < -df['Stdev']
df['Criteria2'] = df['Open'] > df['Moving Average']
df['BUY'] = df['Criteria1'] & df['Criteria2']
df['Pct Change'] = (df['Close'] - df['Open']) / df['Open']
df['Rets'] = df['Pct Change'][df['BUY'] == True]
all_res += df['Rets'].mean()
return all_res
