def IoU(target_labels: np.array, predicted_labels: np.array):
target_labels = target_labels.flatten()
predicted_labels = predicted_labels.flatten()
intersection = np.logical_and(target_labels, predicted_labels)
union = np.logical_or(target_labels, predicted_labels)
iou_score = (np.sum(intersection)+1e-6) / (np.sum(union)+1e-6)
return iou_score
def shortwave(sw_rad: np.array, daylength: np.array, day_of_year: np.array,
tiny_rad_fract: np.array, params: dict) -> np.array:
"""
Disaggregate shortwave radiation down to a subdaily timeseries.
Parameters
----------
sw_rad:
Daily incoming shortwave radiation
daylength:
List of daylength time for each day of year
day_of_year:
Timeseries of index of days since Jan-1
tiny_rad_fract:
Fraction of the daily potential radiation
during a radiation time step defined by SW_RAD_DT
params:
Dictionary of parameters from the MetSim object
Returns
-------
disaggrad:
A sub-daily timeseries of shortwave radiation.
"""
ts = int(params['time_step'])
ts_hourly = float(ts) / cnst.MIN_PER_HOUR
tmp_rad = (sw_rad * daylength) / (cnst.SEC_PER_HOUR * ts_hourly)
n_days = len(tmp_rad)
ts_per_day = int(cnst.HOURS_PER_DAY * cnst.MIN_PER_HOUR / ts)
disaggrad = np.zeros(int(n_days * ts_per_day))
rad_fract_per_day = int(cnst.SEC_PER_DAY / cnst.SW_RAD_DT)
tmp_rad = np.repeat(tmp_rad, rad_fract_per_day)
if params['utc_offset']:
utc_offset = int(
(params['lon'] / cnst.DEG_PER_REV) * rad_fract_per_day)
tiny_rad_fract = np.roll(tiny_rad_fract.flatten(), -utc_offset)
tmp_rad = np.roll(tmp_rad.flatten(), -utc_offset)
tiny_rad_fract = tiny_rad_fract.flatten()
else:
utc_offset = 0
tiny_rad_fract = tiny_rad_fract.flatten()
chunk_size = int(ts * (cnst.SEC_PER_MIN / cnst.SW_RAD_DT))
ts_id = np.repeat(np.arange(ts_per_day), chunk_size)
for day in range(n_days):
# Mask to select out from tiny_rad_fract
radslice = slice((day_of_year[day] - 1) * rad_fract_per_day,
(day_of_year[day]) * rad_fract_per_day)
rad = tiny_rad_fract[radslice]
# Mask to select out time chunk to place disaggregated values into
dslice = slice(int(day * ts_per_day), int((day + 1) * ts_per_day))
# Mask to weight daily solar radiation with
weight_slice = slice(int(day * rad_fract_per_day),
int((day + 1) * rad_fract_per_day))
rad_chunk = np.bincount(ts_id, weights=rad * tmp_rad[weight_slice])
disaggrad[dslice] = rad_chunk
return disaggrad
def get_number_of_tp(y_true: np.array, y_pred: np.array) -> float:
"""
Calculating the number of positive examples which are true positive
:return: accuracy of the positive label
"""
positive_indices = np.where(y_true.flatten() == 1)[0]
return y_pred.flatten()[positive_indices].sum()
def calculate_confusion_matrix_from_arrays(ground_truth: np.array,
prediction: np.array,
num_classes: int) -> np.array:
"""
Calculate confusion matrix for a given set of classes.
if GT value is outside of the [0, num_classes) it is excluded.
Args:
ground_truth:
prediction:
num_classes:
Returns:
"""
# a long 2xn array with each column being a pixel pair
replace_indices = np.vstack((ground_truth.flatten(), prediction.flatten()))
valid_index = replace_indices[0, :] < num_classes
replace_indices = replace_indices[:, valid_index].T
# add up confusion matrix
confusion_matrix, _ = np.histogramdd(
replace_indices,
bins=(num_classes, num_classes),
range=[(0, num_classes), (0, num_classes)],
)
return confusion_matrix.astype(np.uint64)
def calcualte_recall(y_true: np.array, y_pred: np.array) -> float:
"""
Calculating the accuracy of the "1" labels (recall) - querying the indices of the positive values in y_true and check the value
of the prediction in y_pred
:return: accuracy of the positive label
"""
positive_indices = np.where(y_true.flatten() == 1)[0]
return (y_pred.flatten()[positive_indices].sum() / len(positive_indices)).sum()
def image_distance(img1: np.array, img2: np.array) -> float:
"""Calculate the distance between two images for comparism"""
assert img1.shape == img2.shape
img1 = img1.flatten()
img2 = img2.flatten()
# return scipy.spatial.distance.euclidean(img1, img2)
# return scipy.spatial.distance.cosine(img1, img2)
return scipy.spatial.distance.cityblock(img1, img2)
def _flatten_and_concatenate_activations(self,
activations_4_3: numpy.array,
style_array_5_3: numpy.array):
flat_4_3 = activations_4_3.flatten()
flat_5_3 = style_array_5_3.flatten()
# Set precision of target array from float64 to float32 to save some memory (this array is stored for a long time)
concatenated = numpy.concatenate((flat_4_3, flat_5_3))
result_32 = concatenated.astype(dtype=numpy.float32,
casting='same_kind')
return result_32
def inner_prod_fs(a: np.array, b: np.array):
if complex.is_complex(a) and complex.is_complex(b):
return 2 * (a.flatten() @ b.flatten()) - a[:, :, :, 0, :].flatten(
) @ b[:, :, :, 0, :].flatten()
elif complex.is_real(a) and complex.is_real(b):
return 2 * (a.flatten() @ b.flatten()
) - a[:, :, :, 0].flatten() @ b[:, :, :, 0].flatten()
else:
raise NotImplementedError(
'Not implemented for mixed real and complex.')
def update(self, y_t: np.array, y_pred: np.array):
y_t = y_t.flatten()
y_pred = y_pred.flatten()
n_obs = self.model_info.endog.shape[0]
m = self.model
# Residual Calculation
eps_t = y_t - y_pred
m.resid = np.insert(m.resid, n_obs, eps_t, axis=0)
# Add data point
self.model_info.endog = np.insert(self.model_info.endog, n_obs, y_t, axis=0)
def _get_grad(self, x: np.array, y: np.array) -> np.array:
loc = np.arange(y.shape[0])
# Get un-flattened gradient
grad = x[loc, y.flatten()]
grad = -1/grad
# Error signal
d = np.zeros_like(x)
d[loc, y.flatten()] = grad
return d
def save_tdf(filename: str, tdf: np.array, dimx: int, dimy: int, dimz: int,
voxel_size: float, matrix: np.array) -> None:
with open(filename, 'wb') as f:
f.write(struct.pack('I', dimx))
f.write(struct.pack('I', dimy))
f.write(struct.pack('I', dimz))
f.write(struct.pack('f', voxel_size))
f.write(struct.pack("={}f".format(16), *matrix.flatten("F")))
num_elements = dimx * dimy * dimz
f.write(struct.pack("={}f".format(num_elements), *tdf.flatten("F")))
def update(
self,
name :str,
extrinsics :numpy.array,
intrinsics :numpy.array,
correspondences :numpy.array
):
self.data["Viewpoints"].append({
"name" : name,
"extrinsics" : extrinsics.flatten().tolist(),
"intrinsics" : intrinsics.flatten().tolist()
})
def __call__(self, image: np.array) -> Image:
# get image histogram
image_histogram, bins = np.histogram(image.flatten(),
self.number_bins,
density=True)
cdf = image_histogram.cumsum() # cumulative distribution function
cdf = 255 * cdf / cdf[-1] # normalize
# use linear interpolation of cdf to find new pixel values
image_equalized = np.interp(image.flatten(), bins[:-1], cdf)
return Image.fromarray(image_equalized.reshape(image.shape))
def update(self, y_t: np.array, y_pred: np.array):
y_t = y_t.flatten()
y_pred = y_pred.flatten()
n_obs = self.model_info.endog.shape[0]
m = self.model
# Residual Calculation
eps_t = y_t - y_pred
m.resid = np.insert(m.resid, n_obs, eps_t, axis=0)
# Add data point
self.model_info.endog = np.insert(self.model_info.endog,
n_obs,
y_t,
axis=0)
def check_winner(state: np.array, last_move: int, num_rows: int, num_cols: int):
# Borrowed from https://github.com/geoffreyyip/numpy-tictactoe
# TODO: make this more generic so we can play more varied game types
if num_rows != num_cols:
raise ValueError('TODO: handle arbitrary board sizes')
for i in range(0, 3):
# Checks rows and columns for match
rows_win = (state[i, :] == last_move).all()
cols_win = (state[:, i] == last_move).all()
if rows_win or cols_win:
return last_move
diag1_win = (np.diag(state) == last_move).all()
diag2_win = (np.diag(np.fliplr(state)) == last_move).all()
if diag1_win or diag2_win:
# Checks both diagonals for match
return last_move
# Check for draw
if not (state.flatten() == EMPTY).any():
# We have a draw
return DRAW
def calculate_histogram(img_array: np.array) -> (np.array, np.array, np.array):
"""
g1(l) = ∑(l, k=0) pA(k) ⇒ g1(l)−g1(l −1) = pA(l) = hA(l)/NM (l = 1,...,255)
geA(l) = round(255g1(l))
calculate_histogram generates the histogram for an image,
the equalized histogram,
and a new quantized image based on the equalized histogram.
"""
flat = img_array.flatten()
hist = histogram(flat)
cs = cumsum(hist)
nj = (cs - cs.min()) * 255
N = cs.max() - cs.min()
cs = nj / N
cs_casted = cs.astype(np.uint8)
equalized = cs_casted[flat]
img_new = np.reshape(equalized, img_array.shape)
return hist, histogram(equalized), img_new
def _check_Xy(self, X, y: np.array = None) -> [np.array, np.array]:
"""Check X and y to be valid"""
if len(X.shape) != 2:
raise ValueError('X should be 2D (n_samples x n_features)')
if y is not None:
n_samples, n_features = X.shape
if len(y.flatten()) != n_samples:
raise ValueError('number of samples in y is not equal to X')
self.classes_, self.n_classes_ = np.unique(y, return_counts=True)
if len(self.classes_) > 2:
raise NotImplementedError('Just binary class supported'
', multi class not supported yet')
# Get indexes of sorted number of each class
sorted_indexes = np.argsort(self.n_classes_)
# Label of each class
self.minC, self.majC = self.classes_[sorted_indexes]
# Number of each class
self._nMin, self._nMaj = self.n_classes_[sorted_indexes]
# get indexes of minority and majority classes
self._minIndexes = np.where(y != self.majC)[0]
self._majIndexes = np.where(y == self.majC)[0]
# separate X and Y of majority class from whole data
self._majX, self._majY = X[self._majIndexes], y[self._majIndexes]
return X, y
def train_test_split_class(x: np.array,
y: np.array,
split: float,
split_valance=None,
special_case=False):
if split_valance is None:
split_valance = [0.5, 0.5]
if sum(split_valance) != 1:
raise ValueError("The valance should sum to 1")
a_value, b_value = np.unique(y)
indices_a = np.argwhere(y.flatten() == a_value).T.flatten()
indices_b = np.argwhere(y.flatten() == b_value).T.flatten()
# np.random.shuffle(indices_a)
# np.random.shuffle(indices_b)
split_a, split_b = int(
(1 - split * split_valance[0] * 2) * indices_a.shape[0]), int(
(1 - split * split_valance[1] * 2) * indices_b.shape[0])
train_indices_a, val_indices_a = indices_a[:split_a], indices_a[split_a:]
train_indices_b, val_indices_b = indices_b[:split_b], indices_b[split_b:]
x_train, x_val = np.concatenate(
(x[:, train_indices_a], x[:, train_indices_b]),
axis=1), np.concatenate((x[:, val_indices_a], x[:, val_indices_a]),
axis=1)
y_train, y_val = np.concatenate(
(y[:, train_indices_a], y[:, train_indices_b]),
axis=1), np.concatenate((y[:, val_indices_a], y[:, val_indices_a]),
axis=1)
return x_train, x_val, y_train, y_val
def _array_to_stat_list(array: np.array, stat: str) -> list:
list_of_stats = []
# add the results to the lists of values and times
if array.ndim == 1 or array.ndim == 2:
if stat == 'mean':
list_of_stats.append(np.nanmean(array))
elif stat == 'median':
list_of_stats.append(np.nanmedian(array))
elif stat == 'max':
list_of_stats.append(np.nanmax(array))
elif stat == 'min':
list_of_stats.append(np.nanmin(array))
elif stat == 'sum':
list_of_stats.append(np.nansum(array))
elif stat == 'std':
list_of_stats.append(np.nanstd(array))
elif '%' in stat:
list_of_stats.append(
np.nanpercentile(array, int(stat.replace('%', ''))))
elif stat == 'values':
list_of_stats.append(array.flatten().tolist())
else:
raise ValueError(unknown_stat(stat))
elif array.ndim == 3:
for a in array:
list_of_stats += _array_to_stat_list(a, stat)
else:
raise ValueError(
'Too many dimensions in the array. You probably did not mean to do stats like this'
)
return list_of_stats
def get_number_of_positves(y: np.array) -> int:
"""
Calculate the number of positives in an array
:param y:
:return:
"""
return len(np.where(y.flatten() == 1)[0])
def __init__(self, A: np.array=np.zeros((3,4)), b: np.array=np.zeros(3), d: np.array=np.zeros(4)):
self.c = A.flatten()
self.shape = A.shape
self.A_eq = np.hstack([np.identity(self.shape[1])] * self.shape[0])
self.A_ub = block_diag(*[np.ones(self.shape[1])] * self.shape[0])
self.b = b
self.d = d
def totxtfile(self, save_path, value_np: np.array, bit_16_represent=False):
if self.cfg.setup.log_txt:
if bit_16_represent:
np.save(save_path, _cast_bfloat16_then_float32(value_np))
else:
np.savetxt(save_path, value_np.flatten(), fmt='%f', header=str(value_np.shape))
print("----> %s" % save_path)
def forward(self, input:np.array, *args) -> np.array:
'''
forward step
Parameters:
- input: input of shape (batch_size, x_dim, y_dim, z_dim) [numpy.array]
- *args: whether Layer is in training (=True) mode or in prediction mode (=False) [Boolean]
Returns:
- drop_forward: output of shape (batch_size, x_dim, y_dim, z_dim) [numpy.array]
- ac_forward: output after activation function (batch_size, x_dim, y_dim, z_dim) [numpy.array]
'''
self.training, (*_) = args
if self.training:
## get length of flattened input
length = input.flatten().__len__()
## get as much zeros as factor * length
zeros = np.zeros(int(length * self.factor))
## fill the rest with ones
ones = np.ones(length - zeros.__len__())
## concat both, shuffle and reshape to input's shape
scale = np.append(ones, zeros)
np.random.shuffle(scale)
self.scale = scale.reshape(input.shape)
drop_forward = input * self.scale
else:
drop_forward = input.copy()
ac_forward = drop_forward.copy()
return drop_forward, ac_forward
def create_filter(self, initial_detection: np.array):
num_points = initial_detection.shape[0]
dim_z = 2 * num_points
dim_x = 2 * 2 * num_points # We need to accommodate for velocities
filter = KalmanFilter(dim_x=dim_x, dim_z=dim_z)
# State transition matrix (models physics): numpy.array()
filter.F = np.eye(dim_x)
dt = 1 # At each step we update pos with v * dt
filter.F[:dim_z, dim_z:] = dt * np.eye(dim_z)
# Measurement function: numpy.array(dim_z, dim_x)
filter.H = np.eye(
dim_z,
dim_x,
)
# Measurement uncertainty (sensor noise): numpy.array(dim_z, dim_z)
filter.R *= self.R
# Process uncertainty: numpy.array(dim_x, dim_x)
# Don't decrease it too much or trackers pay too little attention to detections
filter.Q[dim_z:, dim_z:] *= self.Q
# Initial state: numpy.array(dim_x, 1)
filter.x[:dim_z] = np.expand_dims(initial_detection.flatten(), 0).T
# Estimation uncertainty: numpy.array(dim_x, dim_x)
filter.P[dim_z:, dim_z:] *= self.P
return filter
def encode_haze(network:MarabouNetwork, image:np.array, epsilon:float, output_index:int) -> MarabouNetwork:
'''Encodes a haze transformation as a Marabou input query
Args:
network (MarabouNetwork): the MarabouNetwork object
image (np.array): The input image
epsilon (float): Amount of transform
output_index (int): Target output node (for the expected class)
Returns:
MarabouNetwork: The MarabouNetwork object with the encoded input query
'''
n_inputs = network.inputVars[0].flatten().shape[0]
n_outputs = network.outputVars[0].flatten().shape[0]
flattened_image = image.flatten()
eps = network.getNewVariable()
network.setLowerBound( eps, 0 )
network.setUpperBound( eps, epsilon )
network.inputVars = np.array([eps])
for i in range(n_inputs):
val = flattened_image[i]
network.addEquality([i, eps], [1, val - 1], val)
for i in range(n_outputs):
if i != output_index:
network.addInequality([network.outputVars[0][i], network.outputVars[0][output_index]], [1, - 1], 0)
return network
def run(self,
likelihood: nirt.likelihood.Likelihood,
theta_active: np.array,
active_ind: np.array) -> \
Tuple[np.array, nirt.likelihood.Likelihood]:
logger = logging.getLogger("Solver.solve_at_resolution")
# Improve theta estimates by Metropolis sweeps.
# A vector of size len(theta_active) * C containing all person parameters.
t = theta_active.flatten()
energy = likelihood.log_likelihood_term(t, active_ind)
theta_estimator = nirt.mcmc.McmcThetaEstimator(likelihood,
self._temperature)
ll = sum(energy)
logger.info("log-likelihood {:.2f}".format(ll))
for sweep in range(self._num_iterations):
ll_old = ll
t, energy = theta_estimator.estimate(t,
active=active_ind,
energy=energy)
ll = sum(energy)
logger.info(
"MCMC sweep {:2d} log-likelihood {:.4f} increase {:.2f} accepted {:.2f}%"
.format(sweep, sum(energy), ll - ll_old,
100 * theta_estimator.acceptance_fraction))
return t.reshape(theta_active.shape) #, likelihood
def __init__(self, x: np.array, y: np.array, n_classes=2, out_features=3):
y = y.flatten()
n_features = x.shape[1]
mean_vectors = []
for i in range(n_classes):
mean_vectors.append(np.mean(x[y == i], axis=0))
s_w = np.zeros((n_features, n_features))
for i, mv in zip(range(n_classes), mean_vectors):
for sample in x[y == i]:
sample, mv = sample.reshape(-1, 1), mv.reshape(n_features, 1)
s_w += (sample - mv).dot((sample - mv).T)
global_mean = np.mean(x, axis=0)
s_b = np.zeros((n_features, n_features))
for i, mean_vec in enumerate(mean_vectors):
n = x[y == i].shape[0]
mean_vec = mean_vec.reshape(-1, 1)
global_mean = global_mean.reshape(-1, 1)
s_b += n * (mean_vec - global_mean).dot((mean_vec - global_mean).T)
eig_vals, eig_vecs = np.linalg.eig(np.linalg.inv(s_w).dot(s_b))
eig_pairs = [(np.abs(eig_vals[i]), eig_vecs[:, i])
for i in range(len(eig_vals))]
eig_pairs = sorted(eig_pairs, key=lambda k: k[0], reverse=True)
self.w = np.hstack(
[eig_pairs[i][1].reshape(-1, 1) for i in range(0, out_features)])
def setup_filter(self, initial_detection: np.array):
initial_detection = validate_points(initial_detection)
dim_x = 2 * 2 * self.num_points # We need to accomodate for velocities
dim_z = 2 * self.num_points
self.dim_z = dim_z
self.filter = KalmanFilter(dim_x=dim_x, dim_z=dim_z)
# State transition matrix (models physics): numpy.array()
self.filter.F = np.eye(dim_x)
dt = 1 # At each step we update pos with v * dt
for p in range(dim_z):
self.filter.F[p, p + dim_z] = dt
# Measurement function: numpy.array(dim_z, dim_x)
self.filter.H = np.eye(
dim_z,
dim_x,
)
# Measurement uncertainty (sensor noise): numpy.array(dim_z, dim_z)
# TODO: maybe we should open this one to the users, as it lets them
# chose between giving more/less importance to the detections
self.filter.R *= 4.0
# Process uncertainty: numpy.array(dim_x, dim_x)
# Don't decrease it too much or trackers pay too little attention to detections
# self.filter.Q[:dim_z, :dim_z] /= 50
self.filter.Q[dim_z:, dim_z:] /= 10
# Initial state: numpy.array(dim_x, 1)
self.filter.x[:dim_z] = np.expand_dims(initial_detection.flatten(), 0).T
# Estimation uncertainty: numpy.array(dim_x, dim_x)
self.filter.P[dim_z:, dim_z:] *= 10.0
def forwards_kinematics(self, current_pose: np.array) -> np.array:
current_pose = current_pose.flatten()
current_pose[2] += current_pose[1] # relative to absolute
theta = current_pose[0]
joint_1 = self._joint_1_length * np.array([[math.cos(
current_pose[1])], [math.sin(current_pose[1])]])
joint_2 = self._joint_2_length * np.array([[math.cos(
current_pose[2])], [math.sin(current_pose[2])]])
end_effector = joint_1 + joint_2
r = np.linalg.norm(end_effector)
end_effector_unit = (end_effector / r).flatten()
# need to subtract half-pi because rho is angle down from z, not angle up from xy
rho = math.pi / 2 - math.atan2(end_effector_unit[1],
end_effector_unit[0])
x = r * math.sin(rho) * math.cos(theta)
y = r * math.sin(rho) * math.sin(theta)
z = r * math.cos(rho)
return np.array([[x], [y], [z]])
def encode_structures(self, cells: np.array) -> np.array: structures = np.zeros(shape=(self.board_width * self.num_planes), dtype=np.uint8) for idx, cell in enumerate(cells.flatten()): if cell.structure is not None: ship = 1 if cell.ship.owner == self.owner else -1 structures[idx] = ship return structures.reshape(self.board_height, self.board_width)
def reachable_states(self, board: np.array) -> Iterable[State]:
flattened = board.flatten()
for i in range(len(flattened)):
if flattened[i] == 0:
state = flattened.copy()
state[i] = self.token
yield tuple(state)
def insert(self, date_str: str, weekly_soil_mois: np.array) -> None:
"""
:param date_str: current data's datetime
:param weekly_soil_mois: data
:return: None
"""
flattened_data = weekly_soil_mois.flatten()
with Connection() as conn:
cur = conn.cursor()
for gid, val in enumerate(flattened_data.tolist()):
val = float('NaN') if val in [-999, -9999] else val
try:
cur.execute(
self.INSERT_SOIL_MOISTURE,
(gid, datetime.datetime.strptime(date_str,
"%Y%m%d"), val))
self.inserted_count += cur.rowcount
conn.commit()
except Exception:
logger.error("error: " + traceback.format_exc())
logger.info(
f'{date_str} finished, total inserted {self.inserted_count}')
cur.close()
