def dist_filter(array, distance):
"""
Iterate through array and remove spurious tracks
:param array: A numpy array of positions (same format as the plotting function)
:param distance: Movement greater than this distance is removed
:return: The filtered array
"""
navg = 5
nfilt = 0
for dim in range(1, array.shape[1], 2):
for npoint in range(navg, array.shape[0] - navg):
point = array[npoint, dim:(dim + 2)]
if np.isnan(point[0]):
continue
else:
# Compute mean positions for last and next num_avg frames
last_set = array[(npoint - navg):npoint, dim:(dim + 2)]
last_set = last_set[np.invert(np.isnan(last_set[:, 0])), :] # wow, numpy is awkward to use
last_mean = last_set.mean(axis=0)
next_set = array[(npoint + 1):(npoint + 6), dim:(dim + 2)]
next_set = next_set[np.invert(np.isnan(next_set[:, 0])), :] # wow, numpy is awkward to use
next_mean = next_set.mean(axis=0)
# If the tracks move more than the threshold, erase it
if (not np.isnan(last_mean[0]) and dist.euclidean(point, last_mean) > distance) or \
(not np.isnan(next_mean[0]) and dist.euclidean(point, next_mean) > distance):
array[npoint, dim:(dim + 2)] = np.nan
nfilt += 1
print(nfilt, ' false tracks removed from the dataset.')
return array
def filter(self, intensity_stack, convert_dtype=False):
I = intensity_stack
I_dtype = I.dtype if not convert_dtype else self.dtype
mask = self._get_mask(I.shape)
I = np.abs(np.array(I).astype(float))
if self.sci_psf is not None:
I = self.sci_qe * conv(np.random.poisson(I).astype(float), self.sci_psf)
if self.psf is not None:
I = conv(I, self.psf)
# convert to well counts
Iel = np.random.poisson(I * self.qe).astype(float)
# add shot noise
Iel += np.abs(np.random.standard_normal(I.shape) * self.shot_noise)
overexposed = Iel >= self.full_well
Iel[overexposed] = self.full_well
Iel[Iel < 0.0] = 0.0
DU = Iel // self.adu
dt = self.dtype
if self.on_limit == "clip":
mx = np.iinfo(dt).max
DU[DU > mx] = mx
DU[np.invert(mask)] = 0.0
mask &= np.invert(overexposed)
return DU.astype(I_dtype), mask
def test_seasonal_get_grouping(self):
dates = get_date_list(dt(2012,4,1),dt(2012,10,31),1)
td = TemporalDimension(value=dates)
## standard seasonal group
calc_grouping = [[6,7,8]]
tg = td.get_grouping(calc_grouping)
self.assertEqual(len(tg.value),1)
selected_months = [s.month for s in td.value[tg.dgroups[0]].flat]
not_selected_months = [s.month for s in td.value[np.invert(tg.dgroups[0])]]
self.assertEqual(set(calc_grouping[0]),set(selected_months))
self.assertFalse(set(not_selected_months).issubset(set(calc_grouping[0])))
## seasons different sizes
calc_grouping = [[4,5,6,7],[8,9,10]]
tg = td.get_grouping(calc_grouping)
self.assertEqual(len(tg.value),2)
self.assertNumpyAll(tg.dgroups[0],np.invert(tg.dgroups[1]))
## crosses year boundary
calc_grouping = [[11,12,1]]
dates = get_date_list(dt(2012,10,1),dt(2013,3,31),1)
td = TemporalDimension(value=dates)
tg = td.get_grouping(calc_grouping)
selected_months = [s.month for s in td.value[tg.dgroups[0]].flat]
self.assertEqual(set(calc_grouping[0]),set(selected_months))
self.assertEqual(tg.value[0],dt(2012,12,16))
## grab real data
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
td = TemporalDimension(value=field.temporal.value_datetime)
tg = td.get_grouping([[3,4,5]])
self.assertEqual(tg.value[0],dt(2005,4,16))
def ScoreSimilarity(ideal, pattern): inverted_ideal = np.invert(ideal) inverted_pattern = np.invert(pattern) # I DON'T THINK THIS IS RIGHT. white = np.sum(np.bitwise_and(ideal, pattern)) black = np.sum(np.bitwise_and(inverted_ideal, inverted_pattern)) return white + black
def _make_image_mask(outlines, pos, res):
"""Aux function
"""
mask_ = np.c_[outlines['mask_pos']]
xmin, xmax = (np.min(np.r_[np.inf, mask_[:, 0]]),
np.max(np.r_[-np.inf, mask_[:, 0]]))
ymin, ymax = (np.min(np.r_[np.inf, mask_[:, 1]]),
np.max(np.r_[-np.inf, mask_[:, 1]]))
inside = _inside_contour(pos, mask_)
outside = np.invert(inside)
outlier_points = pos[outside]
while np.any(outlier_points): # auto shrink
pos *= 0.99
inside = _inside_contour(pos, mask_)
outside = np.invert(inside)
outlier_points = pos[outside]
image_mask = np.zeros((res, res), dtype=bool)
xi_mask = np.linspace(xmin, xmax, res)
yi_mask = np.linspace(ymin, ymax, res)
Xi_mask, Yi_mask = np.meshgrid(xi_mask, yi_mask)
pos_ = np.c_[Xi_mask.flatten(), Yi_mask.flatten()]
inds = _inside_contour(pos_, mask_)
image_mask[inds.reshape(image_mask.shape)] = True
return image_mask, pos
def tri_flat(array, UPPER=True):
"""
Flattens the upper/lower triangle of a square matrix.
Parameters
----------
array : np.ndarray
square matrix
UPPER : boolean
Upper or lower triangle to flatten (defaults to upper). If
the matrix is symmetric, this parameter is unnecessary.
Returns
-------
array : np.ndarray
vector representation of the upper / lower triangle
"""
C = array.shape[0]
if UPPER:
mask = np.asarray(np.invert(np.tri(C,C,dtype=bool)),dtype=float)
else:
mask = np.asarray(np.invert(np.tri(C,C,dtype=bool).transpose()),dtype=float)
x,y = mask.nonzero()
return array[x,y]
def three_dim_pos_bundle(table, key1, key2, key3,
return_complement=False, **kwargs):
"""
Method returns 3d positions of particles in
the standard form of the inputs used by many of the
functions in the `~halotools.mock_observables`.
Parameters
----------
table : data table
`~astropy.table.Table` object
key1, key2, key3: strings
Keys used to access the relevant columns of the data table.
mask : array, optional
array used to apply a mask over the input ``table``. Default is None.
return_complement : bool, optional
If set to True, method will also return the table subset given by the inverse mask.
Default is False.
"""
if 'mask' in kwargs.keys():
mask = kwargs['mask']
x, y, z = table[key1][mask], table[key2][mask], table[key3][mask]
if return_complement is True:
x2, y2, z2 = table[key1][np.invert(mask)], table[key2][np.invert(mask)], table[key3][np.invert(mask)]
return np.vstack((x, y, z)).T, np.vstack((x2, y2, z2)).T
else:
return np.vstack((x, y, z)).T
else:
x, y, z = table[key1], table[key2], table[key3]
return np.vstack((x, y, z)).T
def evaluate(self, X, y, method='f1'):
yhat = self.predict(X)
accurate = y == yhat
positive = np.sum(y == 1)
hatpositive = np.sum(yhat == 1)
tp = np.sum(yhat[accurate] == 1)
#F1 score
if method == 'f1':
recall = 1.*tp/positive if positive > 0 else 0.
precision = 1.*tp/hatpositive if hatpositive > 0 else 0.
f1 = (2.*precision*recall)/(precision+recall) if (precision+recall) > 0 else 0.
return f1
#simple accuracy measure
elif method == 'acc':
return (1.*np.sum(accurate))/len(yhat)
#matthews correlation coefficient
elif method == 'matthews':
tn = np.sum(yhat[accurate] == 0)
fp = np.sum(yhat[np.invert(accurate)] == 1)
fn = np.sum(yhat[np.invert(accurate)] == 0)
denominator = np.sqrt( (tp+fp)*(tp+fn)*(tn+fp)*(tn*fn) )
mat = 1.*((tp*tn)-(fp*fn)) / denominator if denominator > 0 else 0.
return mat
else:
warnings.warn("Wrong evaluation method specified, defaulting to F1 score", RuntimeWarning)
return self.evaluate(X,y)
def archive_human_masks(human_directory, new_directory, work_directory):
'''
For a directory of hand-drawn masks, mask out everything in the accompanying bright-field file except for the worm itself and a 100-pixel surrounding area to save disk space. Also, re-compress all images to maximize compression and space efficiency.
'''
for a_subdir in os.listdir(human_directory):
if os.path.isdir(human_directory + os.path.sep + a_subdir):
folderStuff.ensure_folder(new_directory + os.path.sep + a_subdir)
for a_file in os.listdir(human_directory + os.path.sep + a_subdir):
if a_file.split(' ')[-1] == 'hmask.png':
if not os.path.isfile(new_directory + os.path.sep + a_subdir + os.path.sep + a_file):
print('Up to ' + a_subdir + ' ' + a_file + '.')
my_stem = a_file.split(' ')[0]
my_mask = freeimage.read(human_directory + os.path.sep + a_subdir + os.path.sep + my_stem + ' ' + 'hmask.png')
bf_path = human_directory + os.path.sep + a_subdir + os.path.sep + my_stem + ' ' + 'bf.png'
if os.path.isfile(bf_path):
my_image = freeimage.read(bf_path)
else:
my_image = freeimage.read(bf_path.replace(human_directory, work_directory))
area_mask = my_mask.copy().astype('bool')
distance_from_mask = scipy.ndimage.morphology.distance_transform_edt(np.invert(area_mask)).astype('uint16')
area_mask[distance_from_mask > 0] = True
area_mask[distance_from_mask > 100] = False
my_image[np.invert(area_mask)] = False
freeimage.write(my_image, new_directory + os.path.sep + a_subdir + os.path.sep + my_stem + ' ' + 'bf.png', flags = freeimage.IO_FLAGS.PNG_Z_BEST_COMPRESSION)
freeimage.write(my_mask, new_directory + os.path.sep + a_subdir + os.path.sep + my_stem + ' ' + 'hmask.png', flags = freeimage.IO_FLAGS.PNG_Z_BEST_COMPRESSION)
elif a_file.split('.')[-1] == 'json':
shutil.copyfile(human_directory + os.path.sep + a_subdir + os.path.sep + a_file, new_directory + os.path.sep + a_subdir + os.path.sep + a_file)
return
def extract_local_sparse_matrix(self, target_rank):
logger.debug("Extract local sparse matrix for rank{}".format(target_rank))
t_rank = target_rank
dsts = self.dsts
srcs = self.srcs
wgts = self.wgts
rank_dsts = self.rank_dsts
rank_srcs = self.rank_srcs
t_rank_dsts = rank_dsts == t_rank # bool type array
t_rank_srcs = rank_srcs == t_rank
local_idxs = np.where(t_rank_dsts * t_rank_srcs)[0]
send_idxs = np.where(np.invert(t_rank_dsts) * t_rank_srcs)[0]
recv_idxs = np.where(t_rank_dsts * np.invert(t_rank_srcs))[0]
arr_dict = dict()
arr_dict["spmat_size"] = self.spmat_size
arr_dict["local_dsts"] = dsts[local_idxs]
arr_dict["local_srcs"] = srcs[local_idxs]
arr_dict["local_wgts"] = wgts[local_idxs]
arr_dict["send_ranks"] = rank_dsts[send_idxs]
arr_dict["send_dsts"] = dsts[send_idxs]
arr_dict["send_srcs"] = srcs[send_idxs]
arr_dict["send_wgts"] = wgts[send_idxs]
arr_dict["recv_ranks"] = rank_srcs[recv_idxs]
arr_dict["recv_dsts"] = dsts[recv_idxs]
return arr_dict
def get_combined_calibration(self, nbc_disc, nbc_bulge, split_half=2, names=["m", "c1", "c2"]):
print "Will combine bulge and disc calibration fits."
if split_half==0:
for bias in names:
self.res = arr.add_col(self.res, bias, np.zeros_like(self.res['e1']))
bulge = self.res["is_bulge"].astype(bool)
print "column : %s, bulge : %d/%d, disc : %d/%d"%(bias, self.res[bulge].size, self.res.size, self.res[np.invert(bulge)].size, self.res.size)
try:
self.res[bias][bulge] = nbc_bulge.res[bias][bulge]
except:
import pdb ; pdb.set_trace()
self.res[bias][np.invert(bulge)] = nbc_disc.res[bias][np.invert(bulge)]
else:
com ="""
for i, bias in enumerate(names):
bulge = self.res['is_bulge'].astype(bool)
if i==0: print 'bulge :', self.res[bulge].size, 'disc : ', self.res[np.invert(bulge)].size, 'total : ', self.res.size
self.res = arr.add_col(self.res, bias, np.zeros_like(self.res['e1']))
print 'column : ', bias
self.res[bias][bulge] = nbc_bulge.res[bias][bulge]
self.res[bias][np.invert(bulge)] = nbc_disc.res[bias][np.invert(bulge)]""".replace("res", "res%d"%split_half)
exec(com)
print "done"
def multiple_auc(Y_actual, Y_pred, return_individual=False):
"""
Calculates the averaged ROC for each class
@params:
Y_actual: true values of the labels shape:(n , 1)
Y_pred: predicted values of the labels shape:(n , 1)
@returns:
[0] roc_auc for each class (dict)
[1] averaged_roc amongst classes (float)
"""
uniques = np.unique(Y_actual)
# print uniques
roc_aucs = {}
for label in uniques:
# print label
Y_a = Y_actual.copy()
Y_p = Y_pred.copy()
matches = Y_a == label
Y_a[matches] = -1
Y_a[np.invert(matches)] = 0
Y_a[Y_a == -1] = 1
matches = Y_p == label
Y_p[matches] = -1
Y_p[np.invert(matches)] = 0
Y_p[Y_p == -1] = 1
roc_aucs[label] = roc_auc_score(Y_a, Y_p)
averaged_roc = np.mean(roc_aucs.values())
if return_individual:
return roc_aucs, averaged_roc
else:
return averaged_roc
def _poisson_exp_dense(X, counts, alpha, bias,
beta=None, use_empty_entries=False):
m, n = X.shape
d = euclidean_distances(X)
if use_empty_entries:
mask = (np.invert(np.tri(m, dtype=np.bool)))
else:
mask = np.invert(np.tri(m, dtype=np.bool)) & (counts != 0) & (d != 0)
bias = bias.reshape(-1, 1)
if beta is None:
beta = counts[mask].sum() / (
(d[mask] ** alpha) * (bias * bias.T)[mask]).sum()
g = beta * d ** alpha
g *= bias * bias.T
g = g[mask]
ll = counts[mask] * np.log(beta) + \
alpha * counts[mask] * np.log(d[mask]) + \
counts[mask] * np.log(bias * bias.T)[mask]
ll -= g
# We are trying to maximise, so we need the opposite of the log likelihood
if np.isnan(ll.sum()):
raise ValueError("Objective function is Not a Number")
return - ll.sum()
def enrichment_apply_fn(row, timepoints):
"""
:py:meth:`pandas.DataFrame.apply` apply function for calculating
enrichment scores and r-squared values.
"""
if math.isnan(row[0]):
# not present in input library
score = float("NaN")
r_sq = float("NaN")
else:
row = row.values
ratios = row[np.invert(np.isnan(row))]
times = timepoints[np.invert(np.isnan(row))]
if len(ratios) == 1:
# only present in input library
score = float("NaN")
r_sq = float("NaN")
elif len(ratios) == 2:
# rise over run
score = (ratios[1] - ratios[0]) / (times[1] - times[0])
r_sq = float("NaN")
else:
score, _, r, _, _ = stats.linregress(times, ratios)
r_sq = r ** 2
return pd.Series({'score' : score, 'r_sq' : r_sq})
def baseline_recovery_test(self, model):
baseline_method = getattr(model, 'baseline_'+model._method_name_to_decorate)
baseline_result = baseline_method(halo_table = self.toy_halo_table2)
method = getattr(model, model._method_name_to_decorate)
result = method(halo_table = self.toy_halo_table2)
mask = self.toy_halo_table2['halo_zform_percentile'] >= model._split_ordinates[0]
oldmean = result[mask].mean()
youngmean = result[np.invert(mask)].mean()
baseline_mean = baseline_result.mean()
assert oldmean != youngmean
assert oldmean != baseline_mean
assert youngmean != baseline_mean
param_key = model._method_name_to_decorate + '_assembias_param1'
param = model.param_dict[param_key]
if param > 0:
assert oldmean > youngmean
elif param < 0:
assert oldmean < youngmean
else:
assert oldmean == youngmean
split = model.percentile_splitting_function(halo_table = self.toy_halo_table2)
split = np.where(mask, split, 1-split)
derived_result = split*oldmean
derived_result[np.invert(mask)] = split[np.invert(mask)]*youngmean
derived_mean = derived_result[mask].mean() + derived_result[np.invert(mask)].mean()
baseline_mean = baseline_result.mean()
np.testing.assert_allclose(baseline_mean, derived_mean, rtol=1e-3)
def debug_segmentation(ROI):
# image init, and conversion to gray and then threshold it
img = ROI[:, :, 0:3]
cell_list = []
# Watershed to find individual cells
#img_fine, ret_fine, markers_fine, markers_nuc, joined_mask, cells_to_remove = Klara_test.cell_watershed(img)
cytoplasm_cont, nuclei_mask = Klara_test.cell_watershed(img)
# Put all the objects in the cell_list
#cell_list = modify_cell_list(ROI,ret_fine,markers_fine,markers_nuc,cell_list)
cell_list = Klara_test.modify_cell_list(img, cytoplasm_cont, nuclei_mask)
# Basic classification to RBC and labels these cells
cell_list = RBC_classification(cell_list)
rbc_counter = rbc_cell_extraction(cell_list)
# Also extract image
for cell in cell_list:
if cell.label == "RBC":
subroi = ROI[cell.y:cell.y+cell.h, cell.x:cell.x+cell.w, :]
subroi[:, :, 0] = subroi[:, :, 0]*np.invert(cell.mask.astype(bool))
subroi[:, :, 1] = subroi[:, :, 1]*np.invert(cell.mask.astype(bool))
subroi[:, :, 2] = subroi[:, :, 2]*np.invert(cell.mask.astype(bool))
ROI[cell.y:cell.y+cell.h, cell.x:cell.x+cell.w, :] = subroi
#ROI[cell.mask.astype(int)] = 0
return rbc_counter, ROI
def predict(self, data, modes):
"""predict whether a list of position follows atrain route by detecting
the nearest train stops. Input is the pandas data frame of
measurements and an array of current mode predictions. Returns
an array of predicted modes of the same size as the input data
frame has rows.
"""
# extract lat/lon from data frame
lat = data['WLATITUDE'].values
lon = data['WLONGITUDE'].values
# chunk is a tuple (start_idx, end_idx, mode)
for start_idx, end_idx, _ in ifilter(lambda chunk: chunk[2] in [MODE_CAR, MODE_BUS, MODE_TRAIN],
chunks(modes, include_values=True)):
# test for distance first
lat_seg = lat[start_idx:end_idx]
lon_seg = lon[start_idx:end_idx]
valid_lat_seg = lat_seg[np.where(np.invert(np.isnan(lat_seg)))[0]]
valid_lon_seg = lon_seg[np.where(np.invert(np.isnan(lon_seg)))[0]]
if len(valid_lon_seg) == 0:
continue
# TODO: parameters have to be tuned carefully
is_train = predict_mode_by_location(valid_lat_seg,
valid_lon_seg,
self.train_location_tree,
self.train_location_dict,
self.train_route_dict,
dist_thre = 400,
dist_pass_thres = 7,
num_stops_thre = 3,
dist_pass_thres_perc = 0.7)
#check entry point distance
entry_pt_near = -1
exit_pt_near = -1
if start_idx-1>=0:
if not np.isnan(lat[start_idx-1]):
nearest_station = find_nearest_station(lat[start_idx-1], lon[start_idx-1], self.train_location_tree, self.dist_thres_entry_exit)
if len(nearest_station)!=0:
entry_pt_near = 1
else:
entry_pt_near = 0
if end_idx < len(modes):
if not np.isnan(lat[end_idx]):
nearest_station = find_nearest_station(lat[end_idx],lon[end_idx],
self.train_location_tree,
self.dist_thres_entry_exit)
if len(nearest_station)!=0:
exit_pt_near = 1
else:
exit_pt_near = 0
if is_train or entry_pt_near + exit_pt_near == 2:
modes[start_idx:end_idx] = MODE_TRAIN
else:
modes[start_idx:end_idx] = MODE_CAR
return modes
def calcStats(photoz, specz, verbose=True):
cut = np.logical_and(specz >= 0., photoz >= 0.)
photoz = photoz[cut]
specz = specz[cut]
dz = photoz - specz
sigma_all = np.sqrt( np.sum((dz/(1+specz))**2) / float(len(dz)))
nmad = 1.48 * np.median( np.abs((dz - np.median(dz)) / (1+specz)))
#nmad = 1.48 * np.median( np.abs(dz) / (1+specz))
bias = np.median(dz/(1+specz))
ol1 = (np.abs(dz)/(1+specz) > 0.2 )
OLF1 = np.sum( ol1 ) / float(len(dz))
sigma_ol1 = np.sqrt( np.sum((dz[np.invert(ol1)]/(1+specz[np.invert(ol1)]))**2) / float(len(dz[np.invert(ol1)])))
ol2 = (np.abs(dz)/(1+specz) > 5*nmad )
OLF2 = np.sum( ol2 ) / float(len(dz))
sigma_ol2 = np.sqrt( np.sum((dz[np.invert(ol2)]/(1+specz[np.invert(ol2)]))**2) / float(len(dz[np.invert(ol2)])))
KSscore = ks_2samp(specz, photoz)[0]
if verbose:
print('Sigma_all: {0:.3f}'.format(sigma_all))
print('Sigma_NMAD: {0:.3f}'.format(nmad))
print('Bias: {0:.3f}'.format(bias))
print('OLF: Def1 = {0:.3f} Def2 = {1:0.3f}'.format(OLF1, OLF2))
print('Sigma_OL: Def 1 = {0:.3f} Def2 = {1:0.3f}'.format(sigma_ol1, sigma_ol2))
print('KS: {0:.3f}'.format(KSscore))
return [sigma_all, nmad, bias, OLF1, sigma_ol1, OLF2, sigma_ol2, KSscore]
def _conn_comp_VI(binary_image1, binary_image2,edge_image=True):
# Replaced by _varinfo
if edge_image:
seg1, n1 = _get_segmentation(np.invert(binary_image1))
seg2, n2 = _get_segmentation(np.invert(binary_image2))
else:
seg1, n1 = _get_segmentation(binary_image1)
seg2, n2 = _get_segmentation(binary_image2)
score_ = 0.
n = np.size(binary_image1)
for val in range(1,n1+1):
for val2 in range(1,n2+1):
id1 = seg1==val
id2 = seg2==val2
nx = np.sum(id1)*1.
ny = np.sum(id2)*1.
#rxy = np.sum(np.logical_and(id1,id2))*1./n
rxyn = np.sum(np.logical_and(id1,id2))*1.
if rxyn > 0.:
score_ -= rxyn*(np.log(rxyn/nx)+np.log(rxyn/ny))/n
#if stop:
#print 'n: %d, nx: %f, ny: %f, rxy: %f, score_diff: %f' % (n,nx,ny,rxy,rxy*(np.log(rxy/nx)+np.log(rxy/ny)))
return score_
def segment(self, src):
image = src.ndarray[:]
if self.use_adaptive_threshold:
block_size = 25
markers = threshold_adaptive(image, block_size) * 255
markers = invert(markers)
else:
markers = zeros_like(image)
markers[image < self.threshold_low] = 1
markers[image > self.threshold_high] = 255
elmap = sobel(image, mask=image)
wsrc = watershed(elmap, markers, mask=image)
# elmap = ndimage.distance_transform_edt(image)
# local_maxi = is_local_maximum(elmap, image,
# ones((3, 3))
# )
# markers = ndimage.label(local_maxi)[0]
# wsrc = watershed(-elmap, markers, mask=image)
# fwsrc = ndimage.binary_fill_holes(out)
# return wsrc
if self.use_inverted_image:
out = invert(wsrc)
else:
out = wsrc
# time.sleep(1)
# do_later(lambda:self.show_image(image, -elmap, out))
return out
def measure_autofluorescence(fluorescent_image, worm_mask, time_series):
'''
Measure fluorescence characteristics of a worm at the time corresponding to the information given.
'''
my_fluorescence = fluorescent_image[worm_mask].copy()
(intensity_50, intensity_60, intensity_70, intensity_80, intensity_90, intensity_95, intensity_100) = np.percentile(my_fluorescence, np.array([50, 60, 70, 80, 90, 95, 100]).astype('float64'))
integrated_50 = np.sum(my_fluorescence[my_fluorescence > intensity_50])
integrated_60 = np.sum(my_fluorescence[my_fluorescence > intensity_60])
integrated_70 = np.sum(my_fluorescence[my_fluorescence > intensity_70])
integrated_80 = np.sum(my_fluorescence[my_fluorescence > intensity_80])
integrated_90 = np.sum(my_fluorescence[my_fluorescence > intensity_90])
integrated_95 = np.sum(my_fluorescence[my_fluorescence > intensity_95])
integrated_0 = np.sum(my_fluorescence)
time_series.loc[['intensity_50', 'intensity_60', 'intensity_70', 'intensity_80', 'intensity_90', 'intensity_95', 'intensity_100', 'integrated_50', 'integrated_60', 'integrated_70', 'integrated_80', 'integrated_90', 'integrated_95', 'integrated_0']] = (intensity_50, intensity_60, intensity_70, intensity_80, intensity_90, intensity_95, intensity_100, integrated_50, integrated_60, integrated_70, integrated_80, integrated_90, integrated_95, integrated_0)
over_0_mask = np.zeros(worm_mask.shape).astype('bool')
over_50_mask = np.zeros(worm_mask.shape).astype('bool')
over_60_mask = np.zeros(worm_mask.shape).astype('bool')
over_70_mask = np.zeros(worm_mask.shape).astype('bool')
over_80_mask = np.zeros(worm_mask.shape).astype('bool')
over_90_mask = np.zeros(worm_mask.shape).astype('bool')
over_0_mask[worm_mask] = True
over_50_mask[fluorescent_image > intensity_50] = True
over_50_mask[np.invert(worm_mask)] = False
over_60_mask[fluorescent_image > intensity_60] = True
over_60_mask[np.invert(worm_mask)] = False
over_70_mask[fluorescent_image > intensity_70] = True
over_70_mask[np.invert(worm_mask)] = False
over_80_mask[fluorescent_image > intensity_80] = True
over_80_mask[np.invert(worm_mask)] = False
over_90_mask[fluorescent_image > intensity_90] = True
over_90_mask[np.invert(worm_mask)] = False
colored_areas = color_features([over_0_mask, over_50_mask, over_60_mask, over_70_mask, over_80_mask, over_90_mask])
return (time_series, colored_areas)
def extract_coordinates():
data = np.loadtxt(config_variables.name_of_time_series_promoter_file_for_TSS_start, dtype = str, delimiter = '\t')
plus_strand = data[:, 4] == '+'
minus_strand = np.invert(plus_strand)
promoter_data = np.zeros_like(data).astype(int)[:,:4]
promoter_data[plus_strand, 1] = data[plus_strand, 1].astype(int) - upstream_validation
promoter_data[plus_strand, 2] = data[plus_strand, 1].astype(int) + downstream_validation
promoter_data[minus_strand, 2] = data[minus_strand, 2].astype(int) + upstream_validation
promoter_data[minus_strand, 1] = data[minus_strand, 2].astype(int) - downstream_validation
promoter_data = promoter_data.astype(str)
promoter_data[:, 0] = data[:, 0]
#--------------------
ER_promoters = np.loadtxt("{0}ER_controled_promoters_pindexed.txt".format(temp_output), dtype = str, delimiter = '\t')
Non_ER_promoters = np.loadtxt("{0}Non_ER_controled_promoters_pindexed.txt".format(temp_output), dtype = str, delimiter = '\t')
def un_string(array_to_clean): return np.array(map(lambda x: int(re.findall('\d+', x)[0]), array_to_clean))
ER_promoters_indexes = un_string(ER_promoters[:, 3])
ER_promoters_indexes_mask = np.zeros(len(data), bool)
ER_promoters_indexes_mask[ER_promoters_indexes] = True
promoter_data[np.invert(ER_promoters_indexes_mask), 3] = Non_ER_promoters[:,-1]
promoter_data[ER_promoters_indexes_mask, 3] = ER_promoters[:,-1]
np.savetxt("{0}ER_controled_promoters_pindexed_2.txt".format(temp_output), promoter_data[ER_promoters_indexes_mask], fmt = "%s", delimiter = "\t")
np.savetxt("{0}Non_ER_controled_promoters_pindexed_2.txt".format(temp_output), promoter_data[np.invert(ER_promoters_indexes_mask)], fmt = "%s", delimiter = "\t")
def data_checker_mask(loader):
X, Y = next(loader.get())
X, Y = X[:100].squeeze(), Y[:100]
X, Y = X.transpose([1, 2, 0]), Y.transpose([1, 2, 0])
stride = 60
pos_x = np.arange(0, 600, stride)
pos_y = np.arange(0, 600, stride)
vx, vy = np.meshgrid(pos_x, pos_y)
pos = np.stack([vx, vy]).reshape((2, -1)).transpose([1,0])+stride//2
real, binary = np.zeros((600, 600)), np.zeros((600, 600))
real[patch_interface(pos[:,0], pos[:,1], stride//2)] = X
binary[patch_interface(pos[:,0], pos[:,1], stride//2)] = Y
binary = binary.astype('bool')
# print(real.max(), binary.max(), binary.sum(), binary.size)
img = np.stack([real]*3, axis=2)+500
img[:,:, 0] = img[:,:, 0]*binary
img[:,:, 1] = img[:,:, 1]*np.invert(binary)
img[:,:, 2] = img[:,:, 2]*np.invert(binary)
imwrite(rescale_intensity(img.astype('uint8')), "./save/data_check.png")
def __init__(self, myrank, spfile, ranks, lids):
# Classify Destination, source, weight from the sparse matrix
logger.debug('Classify the meta indices grouped for rank%d'%myrank)
dsts = spfile.dsts
srcs = spfile.srcs
wgts = spfile.wgts
rank_dsts = ranks[dsts] # rank number of destinations
rank_srcs = ranks[srcs] # rank number of sources
myrank_dsts = (rank_dsts == myrank) # bool type array
myrank_srcs = (rank_srcs == myrank)
local_idxs = np.where( myrank_dsts * myrank_srcs )[0]
send_idxs = np.where( np.invert(myrank_dsts) * myrank_srcs )[0]
recv_idxs = np.where( myrank_dsts * np.invert(myrank_srcs) )[0]
dsw_list = [(dsts[i],srcs[i],wgts[i]) for i in local_idxs]
rdsw_list = [(rank_dsts[i],dsts[i],srcs[i],wgts[i]) for i in send_idxs]
rds_list = [(rank_srcs[i],dsts[i],srcs[i]) for i in recv_idxs]
# packaging variables
self.data_dict = data_dict = dict()
data_dict['myrank'] = myrank
data_dict['lids'] = lids
data_dict['dsw_list'] = dsw_list
data_dict['rdsw_list'] = rdsw_list
data_dict['rds_list'] = rds_list
def distance_combinatorics(Dorig,FDR,resolution,n,th,tl,as_str=True,mode=0,res_diff=1.):
D = np.copy(Dorig)
D[((D > tl) & (D < th))] = 1
D[(FDR == 0)] = 1
Dmerged = defaultdict(list)
Dmax = np.zeros(D.shape[1])
for low,high in itertools.combinations(range(D.shape[0]),2):
if resolution[low]/resolution[high] < res_diff:
Dcurrent = np.zeros(D.shape[1])
elif mode == 1:
# positive low, negative high
# zeros where one or both signs incorrect
Dcurrent = (1./(high - low))/(1./4 + 1./(high - low))*((D[high] - 1)*(1 - D[low]))**.5
Dcurrent[np.invert((1 - D[high] < 0)*(1 - D[low] > 0))] = 0
elif mode == 2:
# positive low, positive high
Dcurrent = (1./4 + 1./(high - low))/(1./(high - low))*((1 - D[high])*(1 - D[low]))**.5
Dcurrent[np.invert((1 - D[high] > 0)*(1 - D[low] > 0))] = 0
Dmax = np.array([Dmax,Dcurrent]).max(0)
#Dmax[np.isnan(Dmax)] = 0
del Dcurrent
for i,idx in enumerate(itertools.combinations(xrange(n),2)):
d = Dmax[i]
if d > .0:
if as_str:
Dmerged[idx[0]].append('%d:%f' % (idx[1],d))
Dmerged[idx[1]].append('%d:%f' % (idx[0],d))
else:
Dmerged[idx[0]].append((idx[1],d))
Dmerged[idx[1]].append((idx[0],d))
return Dmerged
def pcols(self, pheno):
'''
Requires a list.
'''
expt_cols = []
pheno_cols = []
if pheno is None:
self._experimentcolumns = self.columns
self._phenocolumns = None
return
if not isinstance(pheno, list):
raise TypeError("A list is required for setting pheno columns")
# Create a column name dict for quick lookups
col_dict = dict(zip(self.columns, range(0, len(self.columns))))
is_pheno = array([c in pheno for c in col_dict])
is_expt = invert(is_pheno)
num_pheno = sum(is_pheno)
num_expt = sum(is_expt)
# Sanity check!
if num_pheno + num_expt != len(self.columns):
raise ValueError("Not all phenotype columns could be found in \
the GenomeFrame.")
# Assign values
if num_pheno > 0:
pheno_cols = self.columns[is_pheno].tolist()
if num_expt > 0:
expt_cols = self.columns[invert(is_pheno)].tolist()
self._phenocolumns = pheno_cols
self._experimentcolumns = expt_cols
def derivative_G(propensities,V,X,w,deter_vector,stoc_positions, positions, valid):
# just the deterministics
X_d = X[deter_vector,:].copy()
temp_eta = np.zeros((np.sum(deter_vector),X.shape[1]))
j = 0
for i in range(len(stoc_positions)):
##pdb.set_trace()
# If x-\nu_i is non zero
if stoc_positions[i] == True:
if np.sum(valid[:,j]) != 0:
#print(" X shape: " + str(X.shape))
#print(" w shape: " + str(w.shape))
#print("test :" + str(map(propensities[i],*X[:,positions[valid[:,j]][:,j]])))
temp_eta[:,valid[:,j]] += (X_d[:,positions[valid[:,j]][:,j]]
- X_d[:,valid[:,j]] +
V[i][deter_vector][:,np.newaxis]
)*map(propensities[i],* X[:,positions[valid[:,j]][:,j]])*w[positions[valid[:,j]][:,j]]
j += 1
else:
temp_eta[:,:] += (V[i][deter_vector][:,np.newaxis])*map(propensities[i],* X)*w
return_X = np.zeros(X.shape)
return_X[deter_vector,:] = temp_eta
return_X[np.invert(deter_vector),:] = X[np.invert(deter_vector),:].copy()
return return_X
def uniform():
bits_to_get = fast_random_bool((self.psize, self.numbit))
pop_part = self.pop_part_rec # if fitness function is too hard, then it could be faster to take best children only for some part of population
# When pop_part = 1.0 it is slower, but based on few tests it's better to leave the children with max fit. Maybe with some probability?
bound = int(self.psize * pop_part)
buff1 = np.empty((1, self.numbit), dtype=int)
buff2 = np.empty((1, self.numbit), dtype=int)
for p_index in range(bound):
buff1[0] = (self.data[self.parents[2 * p_index]] & bits_to_get[p_index]) + (
self.data[self.parents[2 * p_index + 1]] & (np.invert(bits_to_get[p_index]) + 2))
buff2[0] = (self.data[self.parents[2 * p_index + 1]] & bits_to_get[p_index]) + (
self.data[self.parents[2 * p_index]] & (np.invert(bits_to_get[p_index]) + 2))
if self.fitness_function(buff1[0]) > self.fitness_function(buff2[0]):
self.children[p_index] = buff1[0]
else:
self.children[p_index] = buff2[0]
if bound != self.psize:
# choose just first child, not necessarily the best
self.children[bound:self.psize] = (self.data[self.parents[2 * bound::2]] & bits_to_get[
bound:self.psize]) + (
self.data[self.parents[2 * bound + 1::2]] & (
np.invert(bits_to_get[bound:self.psize]) + 2))
del buff1
del buff2
return
def query_by_bagging(X, y, current_model, batch_size, rng, base_model=SVC(C=1, kernel='linear'), n_bags=5, method="KL", D=None):
"""
:param base_model: Model that will be **fitted every iteration**
:param n_bags: Number of bags on which train n_bags models
:param method: 'entropy' or 'KL'
:return:
"""
assert method == 'entropy' or method == 'KL'
eps = 0.0000001
if method == 'KL':
assert hasattr(base_model, 'predict_proba'), "Model with probability prediction needs to be passed to this strategy!"
clfs = BaggingClassifier(base_model, n_estimators=n_bags, random_state=rng)
clfs.fit(X[y.known], y[y.known])
pc = clfs.predict_proba(X[np.invert(y.known)])
# Settles page 17
if method == 'entropy':
pc += eps
fitness = np.sum(pc * np.log(pc), axis=1)
ids = np.argsort(fitness)[:batch_size]
elif method == 'KL':
p = np.array([clf.predict_proba(X[np.invert(y.known)]) for clf in clfs.estimators_])
fitness = np.mean(np.sum(p * np.log(p / pc), axis=2), axis=0)
ids = np.argsort(fitness)[-batch_size:]
return y.unknown_ids[ids], fitness/np.max(fitness)
def rank_S2_predictions(self, forcefields, indices):
norms = {}
means = {}
SDs = {}
for simName in self.S2.keys():
s2 = self.S2[simName]
diff = s2 - self.S2exp
norms[simName] = np.linalg.norm(diff[np.invert(np.isnan(diff))])
means[simName] = diff[np.invert(np.isnan(diff))].mean()
SDs[simName] = diff[np.invert(np.isnan(diff))].std()
data = means
fig = plt.figure()
i = 0
for ff in forcefields:
x = []; y = []
for index in indices:
simName = "{:s}_2IL6_{:d}".format(ff, index)
x.append(i)
y.append(data[simName])
i += 1
plt.plot(x,y, 'o', 'MarkerSize', 2)
plt.show()
def evaluate_sysu(distmat, q_pids, g_pids, q_camids, g_camids, max_rank=20):
"""Evaluation with sysu metric
Key: for each query identity, its gallery images from the same camera view are discarded.
"""
num_q, num_g = distmat.shape
if num_g < max_rank:
max_rank = num_g
print("Note: number of gallery samples is quite small, got {}".format(
num_g))
indices = np.argsort(distmat, axis=1)
pred_label = g_pids[indices]
matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32)
# compute cmc curve for each query
new_all_cmc = []
all_cmc = []
all_AP = []
num_valid_q = 0. # number of valid query
for q_idx in range(num_q):
# get query pid and camid
q_pid = q_pids[q_idx]
q_camid = q_camids[q_idx]
# remove gallery samples that have the same pid and camid with query
order = indices[q_idx]
remove = (q_camid == 3) & (g_camids[order] == 2)
keep = np.invert(remove)
# compute cmc curve
# the cmc calculation is different from standard protocol
# we follow the protocol of the author's released code
new_cmc = pred_label[q_idx][keep]
new_index = np.unique(new_cmc, return_index=True)[1]
new_cmc = [new_cmc[index] for index in sorted(new_index)]
new_match = (new_cmc == q_pid).astype(np.int32)
new_cmc = new_match.cumsum()
new_all_cmc.append(new_cmc[:max_rank])
orig_cmc = matches[q_idx][
keep] # binary vector, positions with value 1 are correct matches
if not np.any(orig_cmc):
# this condition is true when query identity does not appear in gallery
continue
cmc = orig_cmc.cumsum()
cmc[cmc > 1] = 1
all_cmc.append(cmc[:max_rank])
num_valid_q += 1.
# compute average precision
# reference: https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Average_precision
num_rel = orig_cmc.sum()
tmp_cmc = orig_cmc.cumsum()
tmp_cmc = [x / (i + 1.) for i, x in enumerate(tmp_cmc)]
tmp_cmc = np.asarray(tmp_cmc) * orig_cmc
AP = tmp_cmc.sum() / num_rel
all_AP.append(AP)
assert num_valid_q > 0, "Error: all query identities do not appear in gallery"
all_cmc = np.asarray(all_cmc).astype(np.float32)
all_cmc = all_cmc.sum(0) / num_valid_q
new_all_cmc = np.asarray(new_all_cmc).astype(np.float32)
new_all_cmc = new_all_cmc.sum(0) / num_valid_q
mAP = np.mean(all_AP)
return new_all_cmc, mAP
w1 = h
w = h
diff = math.floor((w - w1) / 2)
x = x - diff
cntImg = bwImg[y:y + h, x:x + w]
cntImg1 = cv.copyMakeBorder(cntImg,
round(h / 14),
round(h / 14),
round(w / 14),
round(w / 14),
cv.BORDER_CONSTANT,
value=(255, 255, 255))
invImg = cv.resize(cntImg1, (28, 28))
invImg = np.invert([invImg])
finalImg = invImg.reshape(1, 28, 28, 1)
prediction = model.predict(finalImg)
predictionDict = {}
for i in range(3):
predictionDict["ans" + str(i)] = [
chr(int(f"{np.argmax(prediction)+96}")),
str(f"{((prediction[0][np.argmax(prediction)])*100):.2f}%")
]
prediction[0][np.argmax(prediction)] = 0
for z in predictionDict.keys():
print(f"{(predictionDict[z])[0]} {(predictionDict[z])[1]}")
plt.imshow(finalImg[0])
def state_to_features(game_state: dict) -> np.array:
"""
*This is not a required function, but an idea to structure your code.*
Converts the game state to the input of your model, i.e.
a feature vector.
You can find out about the state of the game environment via game_state,
which is a dictionary. Consult 'get_state_for_agent' in environment.py to see
what it contains.
:param game_state: A dictionary describing the current game board.
:return: np.array
"""
# This is the dict before the game begins and after it ends
if game_state is None:
return None
#--------------------------------------------------------------------------#
# Read game state #
#--------------------------------------------------------------------------#
# extract the field, coin positions and agent information
field = game_state['field']
coins = game_state['coins']
name, score, bomb, location = game_state['self']
bombs = game_state['bombs']
bombs_location = [bomb[0] for bomb in bombs]
enemies = game_state['others']
#print('Field explosion_map: ', np.flip(np.flip(game_state['explosion_map'], axis=0).T,axis=1))
# Copy of field to change
field_ = field.copy()
# Find if bombs are blocking tiles
if len(bombs) > 0:
#bombs_location = [bomb[0] for bomb in bombs]
bombs_ticker = np.array([bomb[1] for bomb in bombs])
# An old bomb blocks the tile
bombs_location_not_self = [
bombs_location[i]
for i in np.where(location not in bombs_location)[0]
]
for i, j in bombs_location_not_self:
field_[i, j] = -1
field_toagent = field_.copy()
# agent movements (top - right - down - left)
area = [(0, -1), (1, 0), (0, 1), (-1, 0)]
if len(enemies) > 0:
enemy_location = [enemy[3] for enemy in enemies]
for eloc in enemy_location:
for direction in area:
neighbor = tuple(map(sum, zip(eloc, direction)))
if location != neighbor:
field_[neighbor[0], neighbor[1]] = -1
# get info for the surroundings of the agent (N-E-S-W)
sur = [tuple(map(sum, zip(location, n))) for n in area]
idx = np.where([s in bombs_location for s in sur])[0]
sur_val = np.array([field[c[0], c[1]] for c in sur])
if len(idx) > 0:
sur_val[idx] = -1
#--------------------------------------------------------------------------#
# Field graphs #
#--------------------------------------------------------------------------#
graph_walkable = make_field_graph(field_)
#print('graph_walkable: ', graph_walkable)
#--------------------------------------------------------------------------#
#--------------------------------------------------------------------------#
# Enemy related features #
#--------------------------------------------------------------------------#
# Distance to all enemies
if len(enemies) > 0:
graph_walkable_to_agent = make_field_graph(field_toagent)
#enemy_location = [enemy[3] for enemy in enemies]
# calculate distance to each enemy
enemy_dist = [
BFS_SP(graph_walkable_to_agent, location, enemy, return_path=False)
for enemy in enemy_location
]
# if path is blocked by a bomb or crate then distance is returned as None
enemy_dist = np.array([1000 if d is None else d for d in enemy_dist])
enemy_reldis = np.array(enemy_location) - np.array(location)[None, ]
idx = np.argsort(enemy_dist)[0]
if enemy_dist[idx] == 1000:
# this only works if tracking 1
enemyf = np.zeros(4)
else:
# Number of targeted enemies
enemies_to_explode = []
for direction in area:
loc = location
for i in range(1, 4):
neighbor = tuple(map(sum, zip(loc, direction)))
if field[neighbor[0], neighbor[1]] == -1:
break
if neighbor in enemy_location:
enemies_to_explode.append(neighbor)
loc = neighbor
enemies_to_explode = np.array(
(len(enemies_to_explode) > 0) * 1) # simplified only 1 and 0
#enemies_to_explode = np.array(len(enemies_to_explode)) # real n of enemies
enemyf = np.hstack((enemy_dist[idx, None], enemy_reldis[idx, :],
enemies_to_explode)).flatten()
else:
enemyf = np.zeros(4)
#--------------------------------------------------------------------------#
# Coin related features #
#--------------------------------------------------------------------------#
# Distance to all coins
if len(coins) > 0:
# calculate distance to each coin
coin_dist = [
BFS_SP(graph_walkable, location, coin, return_path=False)
for coin in coins
]
# if path is blocked by a bomb or crate then distance is returned as None
coin_dist = np.array([1000 if d is None else d for d in coin_dist])
coin_reldis = np.array(coins) - np.array(location)[None, ]
idx = np.argsort(coin_dist)[0]
if coin_dist[idx] == 1000:
# this only works if tracking 1 coind
coinf = np.zeros(3)
#coin_dist[coin_dist == 1000] = 0
else:
coinf = np.hstack(
(coin_dist[idx, None], coin_reldis[idx, :])).flatten()
# coinf, best_coin_pos = look_for_targets_dist(field_ == 0, location, coins)
# coinf[0] = BFS_SP(graph_walkable, location, best_coin_pos, return_path=False)
else:
coinf = np.zeros(3)
#to_fill = trackNcoins*3 - len(coinf)
#coinf = np.hstack((coinf, np.repeat(0, to_fill)))
#print('coinf: ', coinf)
#coinf = np.hstack((coinf, np.repeat(np.nan, to_fill)))
#--------------------------------------------------------------------------#
# Crate related features #
#--------------------------------------------------------------------------#
# Filter our those crates that will explode by an active bomb
# Set crates that will explode to -1
for bomb in bombs_location:
for direction in area:
loc = bomb
for i in range(1, 4):
neighbor = tuple(map(sum, zip(loc, direction)))
if field[neighbor[0], neighbor[1]] == -1:
break
if field[neighbor[0], neighbor[1]] == 1:
field_[neighbor[0], neighbor[1]] = -1
loc = neighbor
# Create graph with paths to all crates
graph_empty_field = make_field_graph(np.invert(field_ >= 0) * 1)
#print('Field target crates -1: ', np.flip(np.flip(field_, axis=0).T,axis=1))
# Distance to 3 closest crates and number of crates that can be blown away at current position
xcrate, ycrate = np.where(field_ == 1)
crates = [(xcrate[i], ycrate[i]) for i in range(len(xcrate))]
if len(crates) > 0:
# Look for closest crate
#print("Target crate:", look_for_targets_dist(field_ >= 0, location, crates))
#cratef = look_for_targets_dist(field_ >= 0, location, crates)
crate_reldis = np.array(crates) - np.array(location)[None, ]
idx = np.argsort(np.sum(np.abs(crate_reldis),
axis=1))[0:6] # look at the 6 closest crates
crate_reldis = crate_reldis[idx]
crates2 = [crates[i] for i in idx]
crate_dist = [
BFS_SP(graph_empty_field, location, crate, return_path=False)
for crate in crates2
]
# if path is blocked by a bomb or crate then distance is returned as None
crate_dist = np.array([1000 if d is None else d for d in crate_dist])
idx = np.argsort(crate_dist)[0] # select the closest crate
if crate_dist[idx] == 1000:
# this only works if tracking 1 coind
cratef = np.zeros(3)
#coin_dist[coin_dist == 1000] = 0
else:
cratef = np.hstack(
(crate_dist[idx, None], crate_reldis[idx, :])).flatten()
else:
cratef = np.zeros(3)
# Number of crates that will explode
crates_to_explode = []
for direction in area:
loc = location
for i in range(1, 4):
neighbor = tuple(map(sum, zip(loc, direction)))
if field[neighbor[0], neighbor[1]] == -1:
break
if neighbor in crates:
crates_to_explode.append(neighbor)
loc = neighbor
cratef = np.hstack((cratef, np.array(len(crates_to_explode))))
#--------------------------------------------------------------------------#
# Bomb related features #
#--------------------------------------------------------------------------#
#trackNbombs = len(game_state['others']) + 1
#trackNbombs = 4
# if the bomb will harm you, this features applies to all boms
# ticker, this feature applies to only one bomb
# relative distance to bomb, aplies to only one bomb
if len(bombs) > 0:
#Relative distance to bomb in X and Y axis
bomb_reldis = np.array(bombs_location) - np.array(location)[None, ]
# Find if the bomb can harm the player
bomb_harm = np.array(
explosion_zone(field, bomb_reldis, bombs_location, location))
bomb_harm = (np.sum(bomb_harm) > 0) * 1
# Features
idx = np.argsort(np.sum(np.abs(bomb_reldis), axis=1))
bombsf = np.hstack(
(bombs_ticker[idx, None], bomb_reldis[idx, :])).flatten()
else:
bombsf = []
bomb_harm = np.zeros(1)
bombav = np.array(game_state['self'][2] *
1) # if the BOMB action is available
if trackBombLoca is True:
to_fill = trackNbombs * 3 - len(bombsf)
if to_fill > 0:
#print('bombsf to_fill:', to_fill)
#print('bombsf nofill:', bombsf)
# we have to choose if nan or a high number
#bombsf = np.hstack((bombsf, np.repeat(np.nan, to_fill)))
bombsf = np.hstack((bombsf, np.repeat(0, to_fill)))
bombsf = np.hstack((bombav, bomb_harm, bombsf))
else:
bombsf = np.hstack((bombav, bomb_harm))
#print('bombsf:', bombsf)
#--------------------------------------------------------------------------#
# Bomb placement features #
#--------------------------------------------------------------------------#
# Find if this is a good location for dropping a bomb
# Also returns the direction of scape
# filter out free fields in agent radius of 4
free_tiles = list(graph_walkable.keys())
tile_dis = np.abs(np.array(free_tiles) - np.array(location)[None, ])
idx = np.where(np.sum(tile_dis <= 4, axis=1) == 2)[0]
free_tiles = [free_tiles[i] for i in idx]
# calculate if they are accessible and if yes, the shortest path
# Returns a list of tupples, each tupple has:
# distance to target, second node in path
free_tile_dist_and_escape = [
BFS_SP(graph_walkable, location, tile) for tile in free_tiles
]
#idx = np.where(np.array(free_tile_dist_and_escape) != None)[0]
idx = np.where([elem != None for elem in free_tile_dist_and_escape])[0]
free_tile_dist_and_escape = [free_tile_dist_and_escape[i] for i in idx]
#print('free_tile_dist_and_escape', free_tile_dist_and_escape)
free_tile_dist = [freetile[0]
for freetile in free_tile_dist_and_escape] # Distance
free_tile_escape = [freetile[1]
for freetile in free_tile_dist_and_escape] # Paths
#print('free_tile_dist', free_tile_dist)
free_tiles = [free_tiles[i] for i in idx]
# long_escapes = np.where(np.array(free_tile_dist) >= 40)[0]
# short_scapes = np.where(np.sum(np.array(free_tiles) == np.array(location), axis=1) == 0)[0]
long_escapes = np.array(free_tile_dist) >= 4 # This is a good spot 4 tiles
short_scapes = np.sum(np.array(free_tiles) == np.array(location),
axis=1) == 0 # This is a good spot for escape route
good_spot = 1 if any(long_escapes) or any(
short_scapes) else 0 # good spot =1, bad spot = 0
good_spot = good_spot if len(crates_to_explode) > 0 or enemyf[
3] > 0 else 0 # if no target on sight is a bad spot
#print('free_tile_escape', free_tile_escape)
#--------------------------------------------------------------------------#
# Bomb escape features #
#--------------------------------------------------------------------------#
# Safe path is a path that you can reach before bomb explodes
# len of path should be less or equal than ticker
# If the end point of the path is the explotion range, then it is not a good path
movement_action = [(0, -1), (1, 0), (0, 1), (-1, 0), (0, 0)]
# Find which routes to a free tile are a trap!
if len(bombs) > 0:
# Filter out bombs out of reach, non threatening
# bombs that are not threatening for any of the 5 possible positions
bad_bombs_idx = []
for i in range(len(bombs_location)):
x = []
for j in range(len(movement_action)):
a_location = tuple(map(sum, zip(location, movement_action[j])))
x.extend(
explosion_zone(field, bomb_reldis[i], [bombs_location[i]],
a_location))
#print('a_location', a_location, 'bombs_location[i]', bombs_location[i], 'danger', x)
# Bad bomb is targetting one of the possible future steps
bad_bombs_idx.append(i) if 1 in x else None
#print('bombs_location', bombs_location, 'bad bombs:', bad_bombs_idx)
# Filter out paths that are not reachable before bomb goes off
# returns a list for each path, 1 Harm, 0 No harm
#print('free_tile_escape:', free_tile_escape)
danger_last_tiles = []
#print("bombs_ticker[i]", bombs_ticker, 'path len', [len(t) for t in free_tile_escape])
for tile_path in free_tile_escape:
x = []
for i in bad_bombs_idx:
issafe = explosion_zone(field, bomb_reldis[i],
[bombs_location[i]], tile_path[-1])
if 0 in issafe and len(tile_path) - 1 <= bombs_ticker[i] + 1:
# you can walk to safe tile before bomb goes off
x.append(0)
#x.append(len(tile_path)-1)
elif 0 in issafe and len(tile_path) - 1 > bombs_ticker[i] + 1:
# "Dead end"
x.append(1)
#x.append(-1)
elif 0 in issafe:
# is safe
x.append(0)
else:
x.append(1)
danger_last_tiles.append(x)
#danger_last_tiles = [explosion_zone(field, bomb_reldis, bombs_location, tile_path[-1]) for tile_path in free_tile_escape]
# if there are no harmful bombs in the last tile
no_danger_last_tiles = np.where([
1 not in danger_last_tile for danger_last_tile in danger_last_tiles
])[0]
good_escape_routes = [
free_tile_escape[i] for i in no_danger_last_tiles
]
# get the next step in the good escape routes
# If the path is len 1, then the best option for this route is to staty still
good_next_tiles = [
route[1] if len(route) > 1 else route[0]
for route in good_escape_routes
]
good_next_tiles = list(
set(good_next_tiles)) # the the unique good tiles
#print('good_next_tiles:', good_next_tiles)
# List of tupples with relative coordinates of next good step
if len(good_next_tiles) > 0:
good_step = list(
map(tuple,
np.array(good_next_tiles) - np.array(location)[None, ]))
good_step = np.array(
[0 if n in good_step else -1 for n in movement_action])
else:
good_step = np.repeat(-1, 5)
#print('good_next_ step features to pos:', good_step)
else:
good_step = np.hstack((np.abs(sur_val) * -1, 0))
#good_step = np.repeat(np.nan, 5)
# sum available spaces at this positions use sur
# This chunk helps to find dead ends:
# values of:
# -1: no good way,
# 0 : good way but dead end
# 1 : good way and free way
# If false values will be just -1 (bad) and 0 (good)
if find_dead_end is True:
# agent movements (top - right - down - left - wait)
#movement_action = [(0,-1), (1,0), (0,1), (-1,0)]
# get info for the surroundings of the agent
field_[location[0], location[1]] = -1 # cannot move back
for i in range(len(good_step) - 1):
if good_step[i] == 0:
future_loc = tuple(map(sum, zip(location, movement_action[i])))
sur = [tuple(map(sum, zip(future_loc, n))) for n in area]
sur_val = np.array([field_[c[0], c[1]] for c in sur])
# 1 if no dead end, 0 if dead end
good_step[i] = (np.sum(np.abs(sur_val) == 0) > 0) * 1
#--------------------------------------------------------------------------#
# Check if new bombs are targetting you but not old ones #
#--------------------------------------------------------------------------#
if len(bombs) > 0:
x = [
bombs_ticker[i] == 3 and location != bombs_location[i]
for i in range(len(bombs_ticker))
]
new_bombs_idx = np.where(x)[0]
old_bombs_idx = np.where(np.invert(x))[0]
if len(new_bombs_idx) > 0:
target_bad_bomb = explosion_zone(
field, bomb_reldis[i],
[bombs_location[i] for i in new_bombs_idx], location)
target_new_bomb = 1 if 1 in target_bad_bomb else 0
if 1 in target_bad_bomb and len(old_bombs_idx) > 0:
target_old_bomb = explosion_zone(
field, bomb_reldis[i],
[bombs_location[i] for i in old_bombs_idx], location)
target_new_bomb = 0 if 1 in target_old_bomb else 1
else:
target_new_bomb = 0
else:
target_new_bomb = 0
#--------------------------------------------------------------------------#
# Return state to features #
#--------------------------------------------------------------------------#
# print('Feature good_step n: ', good_step.shape, good_step)
# print('Feature coinf n: ', coinf.shape, coinf)
# print('Feature cratef n: ', cratef.shape, cratef)
# print('Feature bombsf n: ', bombsf.shape, bombsf)
# print('Feature enemyf n: ', enemyf.shape, enemyf)
features = np.hstack(
(good_step[0:4], coinf, cratef, bombsf, np.array(good_spot),
good_step[4], enemyf, target_new_bomb))
#print('features: ', features)
return features.reshape(1, -1)
plt.xlabel('TF (Hz)')
plt.ylabel('Cluster count')
if save_figures:
# Saving figure
save_name = 'Summary_image_TF_selected_clusters_%s' % (imageID)
os.chdir(save_path)
plt.savefig('%s.png' % save_name, bbox_inches='tight')
print('Figure saved')
#%%
all_epoch_ind = np.arange(np.shape(MaxResp_matrix_without_edge)[1])
non_edge_epoch_ind = np.zeros(np.shape(MaxResp_matrix_without_edge)[1],dtype=bool)
if edge_exists:
for i, iEpoch in enumerate(edge_epochs):
non_edge_epoch_ind = (all_epoch_ind == iEpoch) | non_edge_epoch_ind
non_edge_epoch_ind = np.invert(non_edge_epoch_ind)
selected_cluster_TF_tuning_all = MaxResp_matrix_without_edge[selected_cluster_indices,:]
selected_cluster_TF_tuning_no_edge = selected_cluster_TF_tuning_all[:,non_edge_epoch_ind]
#%%
uniq_freq_nums = len(np.where(np.unique(stimulus_information['epoch_frequency'])>0)[0])
selected_cluster_TF_tuning_no_edge = np.zeros(shape=(len(selected_cluster_indices),uniq_freq_nums))
for ind,iCluster in enumerate(selected_cluster_indices):
curr_max_epoch = maxEpochIdx_matrix_all[iCluster]
current_dir = stimulus_information['epoch_dir'][curr_max_epoch]
current_epoch_type = stimulus_information['stim_type'][curr_max_epoch]
required_epoch_array = \
def extension(self, start_n_dim, start_deg):
""" Take information from spectral sequence class, and calculate
extension coefficients for a given position (start_deg, start_n_dim).
"""
death_radii = self.Hom[self.no_pages-1][start_n_dim][
start_deg].barcode[:, 1]
Hom_reps = local_chains.copy_seq(
self.Hom_reps[self.no_pages - 1][start_n_dim][start_deg])
# bars for extension problem of infty page classes
barcode_extension = np.ones((len(death_radii), 2))
barcode_extension[:, 0] = death_radii
barcode_extension[:, 1] *= self.max_r
# if death_radii is equal to max_r, no need to compute ext coefficients
ext_bool = death_radii < self.max_r
# initialize extension matrices as zero matrices
Sdeg = start_deg
Sn_dim = start_n_dim
for chains in Hom_reps:
self.extensions[start_n_dim][start_deg].append(np.zeros((
self.Hom[self.no_pages-1][Sn_dim][Sdeg].dim,
len(death_radii))))
Sdeg += 1
Sn_dim -= 1
# end for
# if there are no extension coefficients to compute, return
if not np.any(ext_bool):
return
# zero out representatives not in ext_bool
for chains in Hom_reps:
for k, local_ref in enumerate(chains.ref):
if len(local_ref) > 0 and np.any(
np.invert(ext_bool[local_ref])):
chains.coord[k][np.invert(ext_bool[local_ref])] *= 0
# end if
# end for
# end for
# COMPUTE EXTENSION COEFFICIENTS
# go through all diagonal
Sdeg = start_deg
Sn_dim = start_n_dim
for idx, chains in enumerate(Hom_reps):
# lift to infinity page and substract betas
Betas, _ = self.first_page_lift(Sn_dim, Sdeg, chains,
death_radii)
# go up to target_page
Betas, _ = self.lift_to_page(Sn_dim, Sdeg, self.no_pages, Betas,
barcode_extension)
# STORE EXTENSION COEFFICIENTS
self.extensions[start_n_dim][start_deg][idx] = Betas.T
# MODIFY TOTAL COMPLEX REPS using BETAS
if np.any(Betas):
for ext_deg, Schains in enumerate(
self.Hom_reps[self.no_pages - 1][Sn_dim][Sdeg]):
# compute chains using betas and substract to reps
# problem with local sums!!!
local_chains_beta = local_chains.sums(Schains, -Betas)
for k, local_coord in enumerate(
Hom_reps[ext_deg + idx].coord):
local_ref = Hom_reps[ext_deg + idx].ref[k]
if (len(local_ref)) > 0 and (
len(local_chains_beta.ref[k]) > 0):
if not np.array_equal(
local_ref, local_chains_beta.ref[k]):
raise ValueError
Hom_reps[ext_deg + idx].coord[k] = (
local_coord + local_chains_beta.coord[k]
) % self.p
elif len(local_chains_beta.ref[k]) > 0:
Hom_reps[ext_deg + idx].ref[
k] = local_chains_beta.ref[k]
Hom_reps[ext_deg + idx].coord[
k] = local_chains_beta.coord[k]
# end elif
# end for
# end for
# end if
# reduce up to 1st page using gammas
for target_page in range(self.no_pages, 1, -1):
# get coefficients on first page
Betas, _ = self.first_page_lift(Sn_dim, Sdeg, chains,
death_radii)
# go up to target_page
Betas, Gammas = self.lift_to_page(
Sn_dim, Sdeg, target_page, Betas, barcode_extension)
# if lift target_page is nonzero, raise an error
if np.any(Betas):
raise(RuntimeError)
# MODIFY TOTAL COMPLEX REPS using GAMMAS
if np.any(Gammas):
# compute coefficients of Gammas in 1st page
image_classes = np.matmul(
self.Im[target_page-1][Sn_dim][Sdeg].coordinates,
-Gammas.T)
# look for bars that might be already zero
if target_page == 2:
# obtain coefficients for gammas
image_chains = local_chains(
self.nerve_spx_number[Sn_dim])
prev = 0
for nerv_idx, next in enumerate(self.cycle_dimensions[
Sn_dim][Sdeg]):
if prev < next:
image_chains.add_entry(
nerv_idx, range(np.size(Gammas, 0)),
np.matmul(
image_classes[prev:next].T,
self.Hom[0][Sn_dim][nerv_idx][
Sdeg].coordinates.T)
)
prev = next
# end for
else:
image_chains = local_chains.sums(
self.Hom_reps[target_page-2][Sn_dim][Sdeg][0],
image_classes.T
)
# end else
chains += image_chains
# end if
# end for
# lift to first page
Betas, lift_coord = self.first_page_lift(Sn_dim, Sdeg, chains,
death_radii)
# correct sign of lift_coord and trivialise references
lift_coord.minus()
# end for
# if lift to first page is nonzero, raise an error
if np.any(Betas):
raise(RuntimeError)
if Sn_dim > 0:
# compute Cech differential of lift_coord
# and add to current reps
image_chains = self.cech_diff(Sn_dim - 1, Sdeg + 1, lift_coord)
Hom_reps[idx+1] = Hom_reps[idx + 1] + image_chains
# advance reduction position
Sdeg += 1
Sn_dim -= 1
def pca_trace(xinit_in,
spec_min_max=None,
predict=None,
npca=None,
pca_explained_var=99.0,
coeff_npoly=None,
coeff_weights=None,
debug=True,
order_vec=None,
lower=3.0,
upper=3.0,
minv=None,
maxv=None,
maxrej=1,
xinit_mean=None):
"""
Use a PCA model to determine the best object (or slit edge) traces for echelle spectrographs.
Args:
xinit: ndarray, (nspec, norders)
Array of input traces that one wants to PCA model. For object finding this will be the traces for orders where
an object was detected. If an object was not detected on some orders (see ech_objfind), the standard star
(or order boundaries) will be assigned to these orders at the correct fractional slit position, and a joint PCA
fit will be performed to the detected traces and the standard/slit traces.
spec_min_max: float or int ndarray, (2, norders), default=None.
This is a 2-d array which defines the minimum and maximum of each order in the
spectral direction on the detector. This should only be used for echelle spectrographs for which the orders do not
entirely cover the detector, and each order passed in for xinit_in is a succession of orders on the detector.
The code will re-map the traces such that they all have the same length, compute the PCA, and then re-map the orders
back. This improves performanc for echelle spectrographs by removing the nonlinear shrinking of the orders so that
the linear pca operation can better predict the traces. THIS IS AN EXPERIMENTAL FEATURE. INITIAL TESTS WITH
XSHOOTER-NIR INDICATED THAT IT DID NOT IMPROVE PERFORMANCE AND SIMPLY LINEAR EXTRAPOLATION OF THE ORDERS INTO THE
REGIONS THAT ARE NOT ILLUMINATED PERFORMED SIGNIFICANTLY BETTER. DO NOT USE UNTIL FURTHER TESTING IS PERFORMED. IT
COULD HELP WITH OTHER MORE NONLINEAR SPECTROGRAPHS.
predict: ndarray, bool (norders,), default = None
Orders which have True are those that will be predicted by extrapolating the fit of the PCA coefficents for those
orders which have False set in this array. The default is None, which means that the coefficients of all orders
will be fit simultaneously and no extrapolation will be performed. For object finding, we use the standard star
(or slit boundaries) as the input for orders for which a trace is not identified and fit the coefficients of all
simultaneously. Thus no extrapolation is performed. For tracing slit boundaries it may be useful to perform
extrapolations.
npca: int, default = None
number of PCA components to be kept. The maximum number of possible PCA components would be = norders, which is to say
that no PCA compression woulud be performed. For the default of None, npca will be automatically determinedy by
calculating the minimum number of components required to explain 99% (pca_explained_var) of the variance in the different orders.
pca_explained_var: float, default = 99
Amount of explained variance cut used to determine where to truncate the PCA, i.e. to determine npca.
coeff_npoly: int, default = None
Order of polynomial fits used for PCA coefficients fitting. The defualt is None, which means that coeff_noly
will be automatically determined by taking the number of orders into account. PCA components that explain
less variance (and are thus much noiser) are fit with lower order.
coeff_weights (np.ndarray): shape = (norders,), default=None
If input these weights will be used for the polynomial fit to the PCA coefficients. Even if you are predicting
orders and hence only fitting a subset of the orders != norders, the shape of coeff_weights must be norders.
Just give the orders you don't plan to fit a weight of zero. This option is useful for fitting object
traces since the weights can be set to (S/N)^2 of each order.
TODO: Perhaps we should get rid of the predict option and simply allow the user to set the weights of the orders
they want predicted to be zero. That would be more straightforward, but would require a rework of the code.
debug: bool, default = False
Show plots useful for debugging.
Returns:
--------
pca_fit: ndarray, float (nspec, norders)
Array with the same size as xinit, which contains the pca fitted orders.
"""
nspec, norders = xinit_in.shape
if order_vec is None:
order_vec = np.arange(norders, dtype=float)
if predict is None:
predict = np.zeros(norders, dtype=bool)
# use_order = True orders used to predict the predict = True bad orders
use_order = np.invert(predict)
ngood = np.sum(use_order)
if ngood < 2:
msgs.warn(
'There are no good traces to PCA fit. There is probably a bug somewhere. Exiting and returning input traces.'
)
return xinit_in, {}, None, None
if spec_min_max is not None:
xinit = remap_orders(xinit_in, spec_min_max)
else:
xinit = xinit_in
# Take out the mean position of each input trace
if xinit_mean is None:
xinit_mean = np.mean(xinit, axis=0)
xpca = xinit - xinit_mean
xpca_use = xpca[:, use_order].T
pca_full = PCA()
pca_full.fit(xpca_use)
var = np.cumsum(
np.round(pca_full.explained_variance_ratio_, decimals=6) * 100)
npca_full = var.size
if npca is None:
if var[0] >= pca_explained_var:
npca = 1
msgs.info(
'The first PCA component contains more than {:5.3f} of the information'
.format(pca_explained_var))
else:
npca = int(
np.ceil(
np.interp(pca_explained_var, var,
np.arange(npca_full) + 1)))
msgs.info(
'Truncated PCA to contain {:5.3f}'.format(pca_explained_var) +
'% of the total variance. ' +
'Number of components to keep is npca = {:d}'.format(npca))
else:
npca = int(npca)
var_trunc = np.interp(float(npca), np.arange(npca_full) + 1.0, var)
msgs.info('Truncated PCA with npca={:d} components contains {:5.3f}'.
format(npca, var_trunc) + '% of the total variance.')
if npca_full < npca:
msgs.warn(
'Not enough good traces for a PCA fit of the requested dimensionality. The full (non-compressing) PCA has size: '
'npca_full = {:d}'.format(npca_full) +
' is < npca = {:d}'.format(npca))
msgs.warn(
'Using the input trace for now. But you should lower npca <= npca_full'
)
return xinit_in, {}, None, None
if coeff_npoly is None:
coeff_npoly = int(
np.fmin(np.fmax(np.floor(3.3 * ngood / norders), 1.0), 3.0))
# Polynomial coefficient for PCA coefficients
npoly_vec = np.zeros(npca, dtype=int)
# Fit first pca dimension (with largest variance) with a higher order npoly depending on number of good orders.
# Fit all higher dimensions (with lower variance) with a line
# Cascade down and use lower order polynomial for PCA directions that contain less variance
for ipoly in range(npca):
npoly_vec[ipoly] = np.fmax(coeff_npoly - ipoly, 1)
pca = PCA(n_components=npca)
pca_coeffs_use = pca.fit_transform(xpca_use)
pca_vectors = pca.components_
pca_coeffs_new = np.zeros((norders, npca))
fit_dict = {}
# Now loop over the dimensionality of the compression and perform a polynomial fit to
for idim in range(npca):
# Only fit the use_order orders, then use this to predict the others
xfit = order_vec[use_order]
yfit = pca_coeffs_use[:, idim]
ncoeff = npoly_vec[idim]
# Apply a 10% relative error to each coefficient. This performs better than use_mad, since larger coefficients
# will always be considered inliers, if the coefficients vary rapidly with order as they sometimes do.
sigma = np.fmax(0.1 * np.abs(yfit), 0.1)
invvar = utils.inverse(sigma**2)
use_weights = coeff_weights[
use_order] if coeff_weights is not None else None
# TODO Note that we are doing a weighted fit using the coeff_weights, but the rejection is still done
# usnig the ad-hoc invvar created in the line above. I cannot think of a better way.
msk_new, poly_out = utils.robust_polyfit_djs(xfit,
yfit,
ncoeff,
invvar=invvar,
weights=use_weights,
function='polynomial',
maxiter=25,
lower=lower,
upper=upper,
maxrej=maxrej,
sticky=False,
use_mad=False,
minx=minv,
maxx=maxv)
# ToDO robust_poly_fit needs to return minv and maxv as outputs for the fits to be usable downstream
pca_coeffs_new[:, idim] = utils.func_val(poly_out, order_vec,
'polynomial')
fit_dict[str(idim)] = {}
fit_dict[str(idim)]['coeffs'] = poly_out
fit_dict[str(idim)]['minv'] = minv
fit_dict[str(idim)]['maxv'] = maxv
if debug:
# Evaluate the fit
xvec = np.linspace(order_vec.min(), order_vec.max(), num=100)
robust_mask_new = msk_new == 1
plt.plot(xfit,
yfit,
'ko',
mfc='None',
markersize=8.0,
label='pca coeff')
plt.plot(xfit[~robust_mask_new],
yfit[~robust_mask_new],
'r+',
markersize=20.0,
label='robust_polyfit_djs rejected')
plt.plot(xvec,
utils.func_val(poly_out, xvec, 'polynomial'),
ls='-.',
color='steelblue',
label='Polynomial fit of order={:d}'.format(ncoeff))
plt.xlabel('Order Number', fontsize=14)
plt.ylabel('PCA Coefficient', fontsize=14)
plt.title('PCA Fit for Dimension #{:d}/{:d}'.format(
idim + 1, npca))
plt.legend()
plt.show()
pca_model = np.outer(pca.mean_, np.ones(norders)) + (np.dot(
pca_coeffs_new, pca_vectors)).T
# pca_model_mean = np.mean(pca_model,0)
# pca_fit = np.outer(np.ones(nspec), xinit_mean) + (pca_model - pca_model_mean)
# JFH which is correct?
pca_fit = np.outer(np.ones(nspec), xinit_mean) + (pca_model)
if spec_min_max is not None:
pca_out = remap_orders(pca_fit, spec_min_max, inverse=True)
else:
pca_out = pca_fit
return pca_out, fit_dict, pca.mean_, pca_vectors
def iter_tracefit(image,
xinit_in,
ncoeff,
inmask=None,
trc_inmask=None,
fwhm=3.0,
maxdev=2.0,
maxiter=25,
niter=9,
gweight=False,
show_fits=False,
idx=None,
verbose=False,
xmin=None,
xmax=None):
""" Utility routine for object find to iteratively trace and fit. Used by both objfind and ech_objfind
Parameters
----------
image: ndaarray, float
Image of objects to be traced
xinit_in: ndarray, float
Initial guesses for spatial direction trace. This can either be an 2-d array with shape
(nspec, nTrace) array, or a 1-d array with shape (nspec) for the case of a single trace.
ncoeff: int
Order of polynomial fits to trace
Optional Parameter
------------------
inmask: ndarray, bool
Input mask for the image
trc_inmask: ndarray, bool
Input mask for the trace, i.e. places where you know the trace is going to be bad that you always want to mask in the
fits. Same size as xinit_in (nspec, nTrace)
fwhm: float
fwhm width parameter for the flux or gaussian weighted tracing. For flux weighted trace the code does a third
of the iterations with window 1.3*fwhm, a third with 1.1*fwhm, and a third with fwhm. For Gaussian weighted tracing
it uses the fwhm to determine the sigma of the Gausisan which is used for all iterations.
gweight: bool, default = False
If gweight is True the routine does Gaussian weighted tracing, if it is False it will do flux weighted tracing.
Normally the approach is to do a round of flux weighted tracing first, and then refine the traces with Gaussian
weighted tracing.
show_fits: bool, default = False
Will plot the data and the fits.
idx: ndarray of strings, default = None
Array of idx IDs for each object. Used only if show_fits is true for the plotting.
xmin: float, default = None
Lower reference for robust_polyfit polynomial fitting. Default is to use zero
xmax: float, defualt = None
Upper refrence for robust_polyfit polynomial fitting. Default is to use the image size in nspec direction
Returns
-------
xpos: ndarray, float
The output has the same size as xinit_in and contains the fit to the spatial direction of trace for each
object.
Revision History
----------------
23-June-2018 Written by J. Hennawi
"""
if inmask is None:
inmask = np.ones_like(image, dtype=bool)
# Allow for single vectors as input as well:
nspec = xinit_in.shape[0]
if xmin is None:
xmin = 0.0
if xmax is None:
xmax = float(nspec - 1)
# Deal with the possibility of vectors as inputs instead of 2d arrays
if xinit_in.ndim == 1:
nobj = 1
xinit = xinit_in.reshape(nspec, 1)
if trc_inmask is not None:
trc_inmask_out = trc_inmask.reshape(nspec, 1)
else:
trc_inmask_out = np.ones_like(xinit, dtype=bool)
else:
nobj = xinit_in.shape[1]
xinit = xinit_in
if trc_inmask is not None:
trc_inmask_out = trc_inmask
else:
trc_inmask_out = np.ones_like(xinit, dtype=bool)
spec_vec = np.arange(nspec)
if verbose:
msgs.info('Fitting the object traces')
# Iterate flux weighted centroiding
fwhm_vec = np.zeros(niter)
fwhm_vec[0:niter // 3] = 1.3 * fwhm
fwhm_vec[niter // 3:2 * niter // 3] = 1.1 * fwhm
fwhm_vec[2 * niter // 3:] = fwhm
if gweight:
title_text = 'Gaussian Weighted'
else:
title_text = 'Flux Weighted'
xfit1 = np.copy(xinit)
for iiter in range(niter):
if gweight:
xpos1, xerr1 = trace_slits.trace_gweight(
image * inmask,
xfit1,
invvar=inmask.astype(float),
sigma=fwhm / 2.3548)
else:
xpos1, xerr1 = trace_slits.trace_fweight(
image * inmask,
xfit1,
invvar=inmask.astype(float),
radius=fwhm_vec[iiter])
# If a trc_inmask was input, always set the masked values to the initial input crutch. The point is that the crutch
# initially comes from either the standard or the slit boundaries, and if we continually replace it for all iterations
# we will naturally always extraplate the trace to match the shape of a high S/N ratio fit (i.e. either the standard)
# or the flat which was used to determine the slit edges.
xpos1[np.invert(trc_inmask_out)] = xinit[np.invert(trc_inmask_out)]
# Do not do any kind of masking based on xerr1. Trace fitting is much more robust when masked pixels are simply
# replaced by the tracing crutch. We thus do not do weighted fits, i.e. uniform weights, but we set the relative
# weight of the trc_inmask pixels to be lower. This way they still get a say but do not overly influence the fit.
xinvvar = np.ones_like(xpos1.T)
xinvvar[np.invert(trc_inmask_out.T)] = 0.1
pos_set1 = pydl.xy2traceset(
np.outer(np.ones(nobj), spec_vec),
xpos1.T,
#inmask = trc_inmask_out.T,
ncoeff=ncoeff,
maxdev=maxdev,
maxiter=maxiter,
invvar=xinvvar,
xmin=xmin,
xmax=xmax)
xfit1 = pos_set1.yfit.T
# bad pixels have errors set to 999 and are returned to lie on the input trace. Use this only for plotting below
#errmask = (xerr1 > 990.0) # bad pixels have errors set to 999 and are returned to lie on the input trace
outmask = pos_set1.outmask.T
# Plot all the points that were not masked initially
if (show_fits) & (iiter == niter - 1):
for iobj in range(nobj):
# The sum of all these masks adds up to the number of pixels.
inmask_trc = np.invert(
trc_inmask_out[:, iobj]) # masked on the way in
errmask = xerr1[:,
iobj] > 990.0 # masked by fweight or gweight, was set to input trace and still fit
rejmask = np.invert(outmask[:, iobj]) & np.invert(
inmask_trc
) # was good on the way in, masked by the poly fit
nomask = outmask[:, iobj] & np.invert(
errmask
) # was actually fit and not replaced to input trace
plt.plot(spec_vec[nomask],
xpos1[nomask, iobj],
marker='o',
c='k',
markersize=3.0,
linestyle='None',
label=title_text + ' Centroid')
plt.plot(spec_vec,
xinit[:, iobj],
c='g',
zorder=25,
linewidth=2.0,
linestyle='--',
label='initial guess')
plt.plot(spec_vec,
xfit1[:, iobj],
c='red',
zorder=30,
linewidth=2.0,
label='fit to trace')
if np.any(errmask):
plt.plot(spec_vec[errmask],
xfit1[errmask, iobj],
c='blue',
marker='+',
markersize=5.0,
linestyle='None',
zorder=20,
label='masked by tracing, set to init guess')
if np.any(rejmask):
plt.plot(spec_vec[rejmask],
xpos1[rejmask, iobj],
c='cyan',
marker='v',
markersize=5.0,
linestyle='None',
zorder=20,
label='masked by polynomial fit')
if np.any(inmask_trc):
plt.plot(spec_vec[inmask_trc],
xpos1[inmask_trc, iobj],
c='orange',
marker='s',
markersize=3.0,
linestyle='None',
zorder=20,
label='input masked points, not fit')
try:
plt.title(title_text +
' Centroid to object {:s}.'.format(idx[iobj]))
except TypeError:
plt.title(title_text +
' Centroid to object {:d}.'.format(iobj))
plt.ylim((0.995 * xfit1[:, iobj].min(),
1.005 * xfit1[:, iobj].max()))
plt.xlabel('Spectral Pixel')
plt.ylabel('Spatial Pixel')
plt.legend()
plt.show()
# Returns the fit, the actual weighted traces, and the pos_set1 object
return xfit1, xpos1, xerr1, pos_set1
def _render(axes, mark, context):
dimension1 = numpy.ma.column_stack(
[mark._table[key] for key in mark._coordinates[0::2]])
dimension2 = numpy.ma.column_stack(
[mark._table[key] for key in mark._coordinates[1::2]])
if mark._coordinate_axes[0] == "x":
X = axes.project("x", dimension1)
Y = axes.project("y", dimension2)
elif mark._coordinate_axes[0] == "y":
X = axes.project("x", dimension2)
Y = axes.project("y", dimension1)
# group styles with common values
shared_styles, unique_styles = split_styles(mark)
# create root Scatterplot element
mark_xml = xml.SubElement(
root_xml,
"g",
id=context.get_id(mark),
attrib={"class": "toyplot-mark-Scatterplot"},
)
print(xml.tostring(mark_xml))
##
mvectors = [
X.T,
Y.T,
[mark._table[key] for key in mark._marker],
[mark._table[key] for key in mark._mtitle],
]
for x, y, marker, mtitle in zip(*mvectors):
not_null = numpy.invert(
numpy.logical_or(numpy.ma.getmaskarray(x),
numpy.ma.getmaskarray(y)))
# create a Series element to hold all data points
series_xml = xml.SubElement(
mark_xml,
"g",
attrib={
"class": "toyplot-Series",
"style": shared_styles,
},
)
print(xml.tostring(series_xml))
print("")
# subselect markers for missing data
vals = [
itertools.count(step=1),
x[not_null],
y[not_null],
marker[not_null],
mtitle[not_null],
]
for idx, dx, dy, dmarker, dtitle in zip(*vals):
marker = toyplot.marker.create(size=dmarker.size,
shape=dmarker.shape,
mstyle=unique_styles["node"][idx],
lstyle={},
label=dmarker.label)
marker_xml = _draw_marker(
root=series_xml,
marker=marker,
cx=dx,
cy=dy,
extra_class="toyplot-Datum",
title=dtitle,
)
# print xml.tostring(marker_xml)
# print ""
if marker.label:
_draw_text(
root=marker_xml,
text=marker.label,
style=unique_styles['text'][idx],
)
print(xml.tostring(marker_xml))
print("")
def plotLOQRunOrder(targetedData, addCalibration=True, compareBatch=True, title='', savePath=None, figureFormat='png', dpi=72, figureSize=(11, 7)):
"""
Visualise ratio of LLOQ and ULOQ by run order, separated by batch. Option to add barchart that summarises across batch
:param TargetedDataset targetedData: TargetedDataset object
:param bool addCalibration: If ``True`` add calibration samples
:param bool compareBatch: If ``True`` add barchart across batch, separated by SampleType
:param str title: Title for the plot
:param savePath: If ``None`` plot interactively, otherwise save the figure to the path specified
:type savePath: None or str
:param str figureFormat: If saving the plot, use this format
:param int dpi: Plot resolution
:param figureSize: Dimensions of the figure
:type figureSize: tuple(float, float)
:raises ValueError: if targetedData does not satisfy to the TargetedDataset definition for QC
:raises ValueError: if :py:attr:`calibration` does not match the number of batch
"""
# Monitored features are plotted in a different color (own featureMask)
# Count <LLOQ / normal / >ULOQ in features not monitored
# Hatch SP and ER with sampleMask
# If addCalibration, add single color bars (max height)
# If more than one batch, create a subplot for each batch
# If compareBatch, add subplot with batch average values for each subgroup. If not compare batch a single legend is plotted
# Prepare all the data first, find the plotting parameters common to all subplots (x-axis, legend), then plot
# x-axis width delta(min max), bar width and legend are shared across all subplots
# Check dataset is fit for plotting
validDataset = targetedData.validateObject(verbose=False, raiseError=False, raiseWarning=False)
if not validDataset['QC']:
raise ValueError('Import Error: targetedData does not satisfy to the TargetedDataset definition for QC')
# Check addCalibration is possible
if addCalibration:
if isinstance(targetedData.calibration, dict):
if 'Acquired Time' not in targetedData.calibration['calibSampleMetadata'].columns:
addCalibration = False
elif isinstance(targetedData.calibration, list):
if 'Acquired Time' not in targetedData.calibration[0]['calibSampleMetadata'].columns:
addCalibration = False
# Init
batches = sorted((numpy.unique(targetedData.sampleMetadata.loc[:, 'Batch'].values[~numpy.isnan(targetedData.sampleMetadata.loc[:, 'Batch'].values)])).astype(int))
number_of_batch = len(batches)
sns.set_style("ticks", {'axes.linewidth': 0.75})
sns.set_color_codes(palette='deep')
# If more than 1 batch, plot batch summary
if (number_of_batch > 1) & compareBatch:
gs = gridspec.GridSpec(number_of_batch + 1, 5)
nbRow = number_of_batch + 1
else:
gs = gridspec.GridSpec(number_of_batch, 5)
nbRow = number_of_batch
# Set plot size
nbRowByHeight = 3
newHeight = int(numpy.ceil((figureSize[1]/nbRowByHeight) * nbRow))
fig = plt.figure(figsize=(figureSize[0], newHeight), dpi=dpi)
batch_data = []
batch_summary = {'SS': {'LLOQ': [], 'normal': [], 'ULOQ': []}, 'SP': {'LLOQ': [], 'normal': [], 'ULOQ': []}, 'ER': {'LLOQ': [], 'normal': [], 'ULOQ': []}}
color_monitored = 'C7'
color_LLOQ = 'C3'
color_normal = 'C2' # 'white'
color_ULOQ = 'C1'
color_calib = 'C0'
# Prepare and store data for each batch
for i in range(number_of_batch):
out_data = dict()
# Batch sample mask
batchMask = (targetedData.sampleMetadata['Batch'].values == batches[i])
# Define feature mask for Monitored and or not
monitoredFeatureMask = (targetedData.featureMetadata['quantificationType'] == QuantificationType.Monitored).values
quantifiedFeatureMask = (targetedData.featureMetadata['quantificationType'] != QuantificationType.Monitored).values
# Define sample types
SPMask = (targetedData.sampleMetadata['SampleType'].values == SampleType.StudyPool) & (targetedData.sampleMetadata['AssayRole'].values == AssayRole.PrecisionReference) & batchMask
ERMask = (targetedData.sampleMetadata['SampleType'].values == SampleType.ExternalReference) & (targetedData.sampleMetadata['AssayRole'].values == AssayRole.PrecisionReference) & batchMask
# x axis
x = targetedData.sampleMetadata.loc[batchMask, 'Acquired Time'].tolist()
x_SP = targetedData.sampleMetadata.loc[SPMask, 'Acquired Time'].tolist()
x_ER = targetedData.sampleMetadata.loc[ERMask, 'Acquired Time'].tolist()
# make sure x_axis is datetime, if it's already a datetime it will trigger an AttributeError
try:
x = [xtime.to_pydatetime() for xtime in x]
x_SP = [xtime.to_pydatetime() for xtime in x_SP]
x_ER = [xtime.to_pydatetime() for xtime in x_ER]
except AttributeError:
pass
out_data['x'] = x
out_data['x_SP'] = x_SP
out_data['x_ER'] = x_ER
if addCalibration:
# list of calibration expected if more than 1 batch
if isinstance(targetedData.calibration, list) & (number_of_batch != 1):
x_calib = targetedData.calibration[i]['calibSampleMetadata']['Acquired Time'].tolist()
# single calibration if only one batch
elif isinstance(targetedData.calibration, dict) & (number_of_batch == 1):
x_calib = targetedData.calibration['calibSampleMetadata']['Acquired Time'].tolist()
else:
raise ValueError('Calibration does not match the number of batch')
out_data['x_calib'] = x_calib
# y-axis (Subclass values)
# Will plots all sample colors in y_, SP and ER are transparent and cover it
y_monitored = numpy.repeat(sum(monitoredFeatureMask), len(x))
y_LLOQ = (targetedData._intensityData[:, quantifiedFeatureMask][batchMask, :] == -numpy.inf).sum(1)
y_normal = ((targetedData._intensityData[:, quantifiedFeatureMask][batchMask, :] != -numpy.inf) & (targetedData._intensityData[:, quantifiedFeatureMask][batchMask, :] != numpy.inf)).sum(1)
y_ULOQ = (targetedData._intensityData[:, quantifiedFeatureMask][batchMask, :] == numpy.inf).sum(1)
height_total = y_monitored + y_LLOQ + y_normal + y_ULOQ
height_total = numpy.unique(height_total)
if len(height_total) != 1:
raise ValueError('Number of features do not match across samples')
else:
height_total = height_total[0]
y_SP = numpy.repeat(height_total, len(x_SP))
y_ER = numpy.repeat(height_total, len(x_ER))
out_data['y_monitored'] = y_monitored
out_data['y_LLOQ'] = y_LLOQ
out_data['y_normal'] = y_normal
out_data['y_ULOQ'] = y_ULOQ
out_data['y_SP'] = y_SP
out_data['y_ER'] = y_ER
if addCalibration:
y_calib = numpy.repeat(height_total, len(x_calib))
out_data['y_calib'] = y_calib
# Store values to compare across batch
tmp_SPMask = (targetedData.sampleMetadata.loc[batchMask, 'SampleType'].values == SampleType.StudyPool) & (targetedData.sampleMetadata.loc[batchMask, 'AssayRole'].values == AssayRole.PrecisionReference)
tmp_ERMask = (targetedData.sampleMetadata.loc[batchMask, 'SampleType'].values == SampleType.ExternalReference) & (targetedData.sampleMetadata.loc[batchMask, 'AssayRole'].values == AssayRole.PrecisionReference)
tmp_SSMask = numpy.invert(tmp_SPMask | tmp_ERMask)
# check some SS exist
if sum(tmp_SSMask) != 0:
batch_summary['SS']['LLOQ'].append(numpy.mean(y_LLOQ[tmp_SSMask]))
batch_summary['SS']['normal'].append(numpy.mean(y_normal[tmp_SSMask]))
batch_summary['SS']['ULOQ'].append(numpy.mean(y_ULOQ[tmp_SSMask]))
else:
batch_summary['SS']['LLOQ'].append(0)
batch_summary['SS']['normal'].append(0)
batch_summary['SS']['ULOQ'].append(0)
# check some SP exist
if sum(tmp_SPMask) != 0:
batch_summary['SP']['LLOQ'].append(numpy.mean(y_LLOQ[tmp_SPMask]))
batch_summary['SP']['normal'].append(numpy.mean(y_normal[tmp_SPMask]))
batch_summary['SP']['ULOQ'].append(numpy.mean(y_ULOQ[tmp_SPMask]))
else:
batch_summary['SP']['LLOQ'].append(0)
batch_summary['SP']['normal'].append(0)
batch_summary['SP']['ULOQ'].append(0)
# check some ER exist
if sum(tmp_ERMask) != 0:
batch_summary['ER']['LLOQ'].append(numpy.mean(y_LLOQ[tmp_ERMask]))
batch_summary['ER']['normal'].append(numpy.mean(y_normal[tmp_ERMask]))
batch_summary['ER']['ULOQ'].append(numpy.mean(y_ULOQ[tmp_ERMask]))
else:
batch_summary['ER']['LLOQ'].append(0)
batch_summary['ER']['normal'].append(0)
batch_summary['ER']['ULOQ'].append(0)
# Axes parameters
# Width of 1 sample (min difference between 2 samples). Diff is in second, matplotlib width use 1=1day
try:
timeList = sorted(x)
timeDiff = [j - i for i, j in zip(timeList[:-1], timeList[1:])]
acquisitionLength = min(timeDiff).seconds # convert to seconds
except ValueError:
timeDiff = [datetime.timedelta(minutes=15)]
acquisitionLength = 900 # 15 min, if not enough points to calculate the diff
interval = acquisitionLength / (24 * 60 * 60) # acquisition length in day
# width = 0.75*interval
out_data['width'] = 0.6 * interval
# Time range of this batch
if addCalibration:
x_full = x + x_calib
else:
x_full = x
minX = min(x_full) - min(timeDiff) # widen left and right so bars aren't cut
maxX = max(x_full) + min(timeDiff)
out_data['minX'] = minX
out_data['maxX'] = maxX
out_data['delta'] = maxX - minX
# Legend parameters - check legends need it
legend = {'has_SP': False, 'has_ER': False, 'has_calib': False}
if len(x_SP) != 0:
legend['has_SP'] = True
if len(x_ER) != 0:
legend['has_ER'] = True
if addCalibration:
if len(x_calib) != 0:
legend['has_calib'] = True
out_data['legend'] = legend
# Store data
batch_data.append(out_data)
# Get common x-axis parameters
common_width = min([i['width'] for i in batch_data])
common_delta = max([i['delta'] for i in batch_data])
delta = numpy.array([common_delta], dtype="timedelta64[us]")[0] # convert from datetime.timedelta to numpy.timedelta64 #from numpy.timedelta64 "This will be fully compatible with the datetime class of the datetime module of Python only when using a time unit of microseconds"
days = delta.astype('timedelta64[D]')
days = days / numpy.timedelta64(1, 'D')
# check width to add minor tick if <48hr. If >48hr do not tick the weekend
if days < 2:
detailedTick = True
else:
detailedTick = False
# Get common legend parameters
common_has_SP = [i['legend']['has_SP'] for i in batch_data]
common_has_ER = [i['legend']['has_ER'] for i in batch_data]
common_has_calib = [i['legend']['has_calib'] for i in batch_data]
has_SP = (True in common_has_SP)
has_ER = (True in common_has_ER)
has_calib = (True in common_has_calib)
# Plot each batch in its subplot
for i in range(number_of_batch):
# Allow for legend space if not compareBatch
if (number_of_batch > 1) & compareBatch:
ax = plt.subplot(gs[i, :])
else:
ax = plt.subplot(gs[i, :-1])
# Plot
height_cumulative = numpy.zeros(len(batch_data[i]['x']))
# Monitored
p_monitored = ax.bar(batch_data[i]['x'], batch_data[i]['y_monitored'], common_width, color=color_monitored)
height_cumulative += batch_data[i]['y_monitored']
# LLOQ
p_LLOQ = ax.bar(batch_data[i]['x'], batch_data[i]['y_LLOQ'], common_width, bottom=height_cumulative, color=color_LLOQ)
height_cumulative += batch_data[i]['y_LLOQ']
# Normal
p_normal = ax.bar(batch_data[i]['x'], batch_data[i]['y_normal'], common_width, bottom=height_cumulative, color=color_normal, alpha=0.3) # , edgecolor='grey', linewidth=0.2)
height_cumulative += batch_data[i]['y_normal']
# ULOQ
p_ULOQ = ax.bar(batch_data[i]['x'], batch_data[i]['y_ULOQ'], common_width, bottom=height_cumulative, color=color_ULOQ)
height_cumulative += batch_data[i]['y_ULOQ']
# SP
edgecolor_SP = ['black'] * len(batch_data[i]['x_SP'])
p_SP = ax.bar(batch_data[i]['x_SP'], batch_data[i]['y_SP'], common_width, fill=False, edgecolor=edgecolor_SP, hatch='/')
# ER
edgecolor_ER = ['black'] * len(batch_data[i]['x_ER'])
p_ER = ax.bar(batch_data[i]['x_ER'], batch_data[i]['y_ER'], common_width, fill=False, edgecolor=edgecolor_ER, hatch='\\')
# Calibration
if addCalibration:
edgecolor_calib = [color_calib] * len(batch_data[i]['x_calib'])
p_calib = ax.bar(batch_data[i]['x_calib'], batch_data[i]['y_calib'], common_width, color=color_calib, edgecolor=edgecolor_calib)
# Annotate Figure
# Annotate axis
if i == number_of_batch - 1: # only x-tick for the last batch
ax.set_xlabel('Acquisition Date')
ax.set_ylabel('Number of features')
ax.set_xlim(batch_data[i]['minX'], batch_data[i]['minX'] + common_delta) # all batch cover the same range
# ax.set_xticklabels(ax.xaxis.get_majorticklabels(), rotation=45)
# x-axis tick mark
if detailedTick: # Major tick (day) + minor tick (6hr)
majorLoc = WeekdayLocator(byweekday=(MO, TU, WE, TH, FR, SA, SU))
majorFormatter = DateFormatter('%d/%m/%y')
minorLoc = HourLocator(byhour=[6, 12, 18], interval=1)
minorFormatter = DateFormatter('%H:%M')
ax.xaxis.set_major_locator(majorLoc)
ax.xaxis.set_major_formatter(majorFormatter)
ax.xaxis.set_minor_locator(minorLoc)
ax.xaxis.set_minor_formatter(minorFormatter)
else: # Major tick (weekday) only
majorLoc = WeekdayLocator(byweekday=(MO, TU, WE, TH, FR))
majorFormatter = DateFormatter('%d/%m/%y')
ax.xaxis.set_major_locator(majorLoc)
ax.xaxis.set_major_formatter(majorFormatter)
# Legend
legendBox = []
legendText = []
# basic color
legendBox.extend([p_monitored[0], p_LLOQ[0], p_normal[0], p_ULOQ[0]])
legendText.extend(['Monitored', '<LLOQ', 'Normal', '>ULOQ'])
if has_SP:
legendBox.append(p_SP[0])
legendText.append('SP')
if has_ER:
legendBox.append(p_ER[0])
legendText.append('ER')
if has_calib:
legendBox.append(p_calib[0])
legendText.append('Calibration')
# Legend to the right only if single batch, or multibatch without compareBatch. Title in multiBatch
if (number_of_batch == 1):
ax.legend(legendBox, legendText, loc='upper left', bbox_to_anchor=(1, 1))
elif (i == 0) & (not compareBatch):
ax.legend(legendBox, legendText, loc='upper left', bbox_to_anchor=(1, 1))
ax.set_title('Batch ' + str(batches[i]))
else:
ax.set_title('Batch ' + str(batches[i]))
# end of Batch subplots
# Summary plot across batch
# If more than 1 batch, plot batch summary
if (number_of_batch > 1) & compareBatch:
x_summary = batches
empty_sampleType = {'LLOQ': numpy.repeat(0, number_of_batch).tolist(),
'normal': numpy.repeat(0, number_of_batch).tolist(),
'ULOQ': numpy.repeat(0, number_of_batch).tolist()}
xi_pos = 0
width = 0.9
# SS
if batch_summary['SS'] != empty_sampleType:
axSS = plt.subplot(gs[number_of_batch, xi_pos])
axSS.bar(x_summary, batch_summary['SS']['LLOQ'], width, color=color_LLOQ)
axSS.bar(x_summary, batch_summary['SS']['normal'], width, color=color_normal, alpha=0.3, bottom=batch_summary['SS']['LLOQ'])
axSS.bar(x_summary, batch_summary['SS']['ULOQ'], width, color=color_ULOQ, bottom=[sum(x) for x in zip(batch_summary['SS']['LLOQ'], batch_summary['SS']['normal'])])
axSS.set_xlabel('Batch')
axSS.set_ylabel('Number of features')
axSS.set_title('SS')
xi_pos += 1
# SP
if batch_summary['SP'] != empty_sampleType:
axSP = plt.subplot(gs[number_of_batch, xi_pos])
axSP.bar(x_summary, batch_summary['SP']['LLOQ'], width, color=color_LLOQ)
axSP.bar(x_summary, batch_summary['SP']['normal'], width, color=color_normal, alpha=0.3, bottom=batch_summary['SP']['LLOQ'])
axSP.bar(x_summary, batch_summary['SP']['ULOQ'], width, color=color_ULOQ, bottom=[sum(x) for x in zip(batch_summary['SP']['LLOQ'], batch_summary['SP']['normal'])])
axSP.set_xlabel('Batch')
axSP.set_title('SP')
xi_pos += 1
# ER
if batch_summary['ER'] != empty_sampleType:
axER = plt.subplot(gs[number_of_batch, xi_pos])
axER.bar(x_summary, batch_summary['ER']['LLOQ'], width, color=color_LLOQ)
axER.bar(x_summary, batch_summary['ER']['normal'], width, color=color_normal, alpha=0.3, bottom=batch_summary['ER']['LLOQ'])
axER.bar(x_summary, batch_summary['ER']['ULOQ'], width, color=color_ULOQ, bottom=[sum(x) for x in zip(batch_summary['ER']['LLOQ'], batch_summary['ER']['normal'])])
axER.set_xlabel('Batch')
axER.set_title('ER')
xi_pos += 1
# Legend
axLeg = plt.subplot(gs[number_of_batch, 4])
axLeg.set_axis_off()
axLeg.legend(legendBox, legendText, loc='center')
# Title and layout
fig.suptitle(title)
fig.tight_layout()
# Save or output
if savePath:
plt.savefig(savePath, bbox_inches='tight', format=figureFormat, dpi=dpi)
plt.close()
else:
plt.show()
mic2mic_distmat[channel_mica, channel_micb] = distance
#%% make the matrix symmetric across the diagonal the 'tedious' way:
upper_or_lower = {0: 'lower', 1: 'upper'}
for i in range(mic2mic_distmat.shape[0]):
for j in range(mic2mic_distmat.shape[0]):
lower = mic2mic_distmat[i, j]
upper = mic2mic_distmat[j, i]
are_nans = [np.isnan(lower), np.isnan(upper)]
if sum(are_nans) == 0:
pass
elif sum(are_nans) == 1:
non_nan_value = int(np.argwhere(np.invert(are_nans)))
if upper_or_lower[non_nan_value] == 'lower':
mic2mic_distmat[j, i] = mic2mic_distmat[i, j]
elif upper_or_lower[non_nan_value] == 'upper':
mic2mic_distmat[i, j] = mic2mic_distmat[j, i]
elif sum(are_nans) == 2:
pass
#%%
distmat = pd.DataFrame(mic2mic_distmat)
distmat.columns = ['channel' + str(each + 1) for each in range(len(good_mics))]
distmat.index = ['channel' + str(each + 1) for each in range(len(good_mics))]
#%% Write the distance matrix -- commented out for consistency
# distmat.to_csv('mic2mic_RAW_2018-08-17.csv',na_rep='NaN')
def create_min_ndvi_mosaic(dataset_in,
clean_mask=None,
no_data=-9999,
dtype=None,
intermediate_product=None,
**kwargs):
"""
Method for calculating the pixel value for the min ndvi value.
Parameters
----------
dataset_in: xarray.Dataset
A dataset retrieved from the Data Cube; should contain:
coordinates: time, latitude, longitude
variables: variables to be mosaicked (e.g. red, green, and blue bands)
clean_mask: np.ndarray
An ndarray of the same shape as `dataset_in` - specifying which values to mask out.
If no clean mask is specified, then all values are kept during compositing.
no_data: int or float
The no data value.
dtype: str or numpy.dtype
A string denoting a Python datatype name (e.g. int, float) or a NumPy dtype (e.g.
np.int16, np.float32) to convert the data to.
Returns
-------
dataset_out: xarray.Dataset
Compositited data with the format:
coordinates: latitude, longitude
variables: same as dataset_in
"""
dataset_in = dataset_in.copy(deep=True)
# Default to masking nothing.
if clean_mask is None:
clean_mask = create_default_clean_mask(dataset_in)
# Save dtypes because masking with Dataset.where() converts to float64.
band_list = list(dataset_in.data_vars)
dataset_in_dtypes = {}
for band in band_list:
dataset_in_dtypes[band] = dataset_in[band].dtype
# Mask out clouds and scan lines.
dataset_in = dataset_in.where((dataset_in != -9999) & clean_mask)
if intermediate_product is not None:
dataset_out = intermediate_product.copy(deep=True)
else:
dataset_out = None
time_slices = range(len(dataset_in.time))
for timeslice in time_slices:
dataset_slice = dataset_in.isel(time=timeslice).drop('time')
ndvi = (dataset_slice.nir - dataset_slice.red) / (dataset_slice.nir +
dataset_slice.red)
ndvi.values[np.invert(clean_mask)[timeslice, ::]] = 1000000000
dataset_slice['ndvi'] = ndvi
if dataset_out is None:
dataset_out = dataset_slice.copy(deep=True)
utilities.clear_attrs(dataset_out)
else:
for key in list(dataset_slice.data_vars):
dataset_out[key].values[
dataset_slice.ndvi.values <
dataset_out.ndvi.values] = dataset_slice[key].values[
dataset_slice.ndvi.values < dataset_out.ndvi.values]
# Handle datatype conversions.
dataset_out = restore_or_convert_dtypes(dtype, band_list,
dataset_in_dtypes, dataset_out,
no_data)
return dataset_out
def generate(self,
x: np.ndarray,
y: Optional[np.ndarray] = None,
**kwargs) -> np.ndarray:
"""
Generate adversarial samples and return them in an array.
:param x: An array with the original inputs.
:param y: Target values (class labels) one-hot-encoded of shape `(nb_samples, nb_classes)` or indices of shape
(nb_samples,). Only provide this parameter if you'd like to use true labels when crafting adversarial
samples. Otherwise, model predictions are used as labels to avoid the "label leaking" effect
(explained in this paper: https://arxiv.org/abs/1611.01236). Default is `None`.
:param mask: An array with a mask broadcastable to input `x` defining where to apply adversarial perturbations.
Shape needs to be broadcastable to the shape of x and can also be of the same shape as `x`. Any
features for which the mask is zero will not be adversarially perturbed.
:type mask: `np.ndarray`
:return: An array holding the adversarial examples.
"""
mask = kwargs.get("mask")
y = check_and_transform_label_format(y, self.estimator.nb_classes)
if y is None:
if self.targeted:
raise ValueError(
"Target labels `y` need to be provided for a targeted attack."
)
y = get_labels_np_array(
self.estimator.predict(x, batch_size=self.batch_size)).astype(
np.int32)
if self.estimator.nb_classes == 2 and y.shape[1] == 1:
raise ValueError(
"This attack has not yet been tested for binary classification with a single output classifier."
)
x_adv = x.astype(ART_NUMPY_DTYPE)
for _ in trange(max(1, self.nb_random_init),
desc="AutoPGD - restart",
disable=not self.verbose):
# Determine correctly predicted samples
y_pred = self.estimator.predict(x_adv)
if self.targeted:
sample_is_robust = np.argmax(y_pred, axis=1) != np.argmax(
y, axis=1)
elif not self.targeted:
sample_is_robust = np.argmax(y_pred,
axis=1) == np.argmax(y, axis=1)
if np.sum(sample_is_robust) == 0:
break
x_robust = x_adv[sample_is_robust]
y_robust = y[sample_is_robust]
x_init = x[sample_is_robust]
n = x_robust.shape[0]
m = np.prod(x_robust.shape[1:]).item()
random_perturbation = (random_sphere(
n, m, self.eps,
self.norm).reshape(x_robust.shape).astype(ART_NUMPY_DTYPE))
x_robust = x_robust + random_perturbation
if self.estimator.clip_values is not None:
clip_min, clip_max = self.estimator.clip_values
x_robust = np.clip(x_robust, clip_min, clip_max)
perturbation = projection(x_robust - x_init, self.eps, self.norm)
x_robust = x_init + perturbation
# Compute perturbation with implicit batching
for batch_id in trange(
int(np.ceil(x_robust.shape[0] / float(self.batch_size))),
desc="AutoPGD - batch",
leave=False,
disable=not self.verbose,
):
self.eta = 2 * self.eps_step
batch_index_1, batch_index_2 = batch_id * self.batch_size, (
batch_id + 1) * self.batch_size
x_k = x_robust[batch_index_1:batch_index_2].astype(
ART_NUMPY_DTYPE)
x_init_batch = x_init[batch_index_1:batch_index_2].astype(
ART_NUMPY_DTYPE)
y_batch = y_robust[batch_index_1:batch_index_2]
p_0 = 0
p_1 = 0.22
var_w = [p_0, p_1]
while True:
p_j_p_1 = var_w[-1] + max(var_w[-1] - var_w[-2] - 0.03,
0.06)
if p_j_p_1 > 1:
break
var_w.append(p_j_p_1)
var_w = [math.ceil(p * self.max_iter) for p in var_w]
eta = self.eps_step
self.count_condition_1 = 0
for k_iter in trange(self.max_iter,
desc="AutoPGD - iteration",
leave=False,
disable=not self.verbose):
# Get perturbation, use small scalar to avoid division by 0
tol = 10e-8
# Get gradient wrt loss; invert it if attack is targeted
grad = self.estimator.loss_gradient(
x_k, y_batch) * (1 - 2 * int(self.targeted))
# Apply norm bound
if self.norm in [np.inf, "inf"]:
grad = np.sign(grad)
elif self.norm == 1:
ind = tuple(range(1, len(x_k.shape)))
grad = grad / (np.sum(
np.abs(grad), axis=ind, keepdims=True) + tol)
elif self.norm == 2:
ind = tuple(range(1, len(x_k.shape)))
grad = grad / (np.sqrt(
np.sum(np.square(grad), axis=ind, keepdims=True)) +
tol)
assert x_k.shape == grad.shape
perturbation = grad
if mask is not None:
perturbation = perturbation * (
mask.astype(ART_NUMPY_DTYPE))
# Apply perturbation and clip
z_k_p_1 = x_k + eta * perturbation
if self.estimator.clip_values is not None:
clip_min, clip_max = self.estimator.clip_values
z_k_p_1 = np.clip(z_k_p_1, clip_min, clip_max)
if k_iter == 0:
x_1 = z_k_p_1
perturbation = projection(x_1 - x_init_batch, self.eps,
self.norm)
x_1 = x_init_batch + perturbation
f_0 = self.estimator.compute_loss(x=x_k,
y=y_batch,
reduction="mean")
f_1 = self.estimator.compute_loss(x=x_1,
y=y_batch,
reduction="mean")
self.eta_w_j_m_1 = eta
self.f_max_w_j_m_1 = f_0
if f_1 >= f_0:
self.f_max = f_1
self.x_max = x_1
self.x_max_m_1 = x_init_batch
self.count_condition_1 += 1
else:
self.f_max = f_0
self.x_max = x_k.copy()
self.x_max_m_1 = x_init_batch
# Settings for next iteration k
x_k_m_1 = x_k.copy()
x_k = x_1
else:
perturbation = projection(z_k_p_1 - x_init_batch,
self.eps, self.norm)
z_k_p_1 = x_init_batch + perturbation
alpha = 0.75
x_k_p_1 = x_k + alpha * (z_k_p_1 - x_k) + (
1 - alpha) * (x_k - x_k_m_1)
if self.estimator.clip_values is not None:
clip_min, clip_max = self.estimator.clip_values
x_k_p_1 = np.clip(x_k_p_1, clip_min, clip_max)
perturbation = projection(x_k_p_1 - x_init_batch,
self.eps, self.norm)
x_k_p_1 = x_init_batch + perturbation
f_k_p_1 = self.estimator.compute_loss(x=x_k_p_1,
y=y_batch,
reduction="mean")
if f_k_p_1 == 0.0:
x_k = x_k_p_1.copy()
break
if (not self.targeted and f_k_p_1 > self.f_max) or (
self.targeted and f_k_p_1 < self.f_max):
self.count_condition_1 += 1
self.x_max = x_k_p_1
self.x_max_m_1 = x_k
self.f_max = f_k_p_1
if k_iter in var_w:
rho = 0.75
condition_1 = self.count_condition_1 < rho * (
k_iter - var_w[var_w.index(k_iter) - 1])
condition_2 = self.eta_w_j_m_1 == eta and self.f_max_w_j_m_1 == self.f_max
if condition_1 or condition_2:
eta = eta / 2
x_k_m_1 = self.x_max_m_1
x_k = self.x_max
else:
x_k_m_1 = x_k
x_k = x_k_p_1.copy()
self.count_condition_1 = 0
self.eta_w_j_m_1 = eta
self.f_max_w_j_m_1 = self.f_max
else:
x_k_m_1 = x_k
x_k = x_k_p_1.copy()
y_pred_adv_k = self.estimator.predict(x_k)
if self.targeted:
sample_is_not_robust_k = np.invert(
np.argmax(y_pred_adv_k, axis=1) != np.argmax(y_batch,
axis=1))
elif not self.targeted:
sample_is_not_robust_k = np.invert(
np.argmax(y_pred_adv_k, axis=1) == np.argmax(y_batch,
axis=1))
x_robust[batch_index_1:batch_index_2][
sample_is_not_robust_k] = x_k[sample_is_not_robust_k]
x_adv[sample_is_robust] = x_robust
return x_adv
def model_eval(X,y):
batch =64
epochs = 20
rep = 1 # K fold procedure can be repeated multiple times
Kfold = 5
Ntrain = 8528 # number of recordings on training set
Nsamp = int(Ntrain/Kfold) # number of recordings to take as validation
# Need to add dimension for training
X = np.expand_dims(X, axis=2)
classes = ['A', 'N', 'O', '~']
Nclass = len(classes)
cvconfusion = np.zeros((Nclass,Nclass,Kfold*rep))
cvscores = []
counter = 0
# repetitions of cross validation
for r in range(rep):
print("Rep %d"%(r+1))
# cross validation loop
for k in range(Kfold):
print("Cross-validation run %d"%(k+1))
# Callbacks definition
callbacks = [
# Early stopping definition
EarlyStopping(monitor='val_loss', patience=3, verbose=1),
# Decrease learning rate by 0.1 factor
AdvancedLearnignRateScheduler(monitor='val_loss', patience=1,verbose=1, mode='auto', decayRatio=0.1),
# Saving best model
ModelCheckpoint('weights-best_k{}_r{}.hdf5'.format(k,r), monitor='val_loss', save_best_only=True, verbose=1),
]
# Load model
model = ResNet_model(WINDOW_SIZE)
# split train and validation sets
idxval = np.random.choice(Ntrain, Nsamp,replace=False)
idxtrain = np.invert(np.in1d(range(X_train.shape[0]),idxval))
ytrain = y[np.asarray(idxtrain),:]
Xtrain = X[np.asarray(idxtrain),:,:]
Xval = X[np.asarray(idxval),:,:]
yval = y[np.asarray(idxval),:]
# Train model
model.fit(Xtrain, ytrain,
validation_data=(Xval, yval),
epochs=epochs, batch_size=batch,callbacks=callbacks)
# Evaluate best trained model
model.load_weights('weights-best_k{}_r{}.hdf5'.format(k,r))
ypred = model.predict(Xval)
ypred = np.argmax(ypred,axis=1)
ytrue = np.argmax(yval,axis=1)
cvconfusion[:,:,counter] = confusion_matrix(ytrue, ypred)
F1 = np.zeros((4,1))
for i in range(4):
F1[i]=2*cvconfusion[i,i,counter]/(np.sum(cvconfusion[i,:,counter])+np.sum(cvconfusion[:,i,counter]))
print("F1 measure for {} rhythm: {:1.4f}".format(classes[i],F1[i,0]))
cvscores.append(np.mean(F1)* 100)
print("Overall F1 measure: {:1.4f}".format(np.mean(F1)))
K.clear_session()
gc.collect()
config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
K.set_session(sess)
counter += 1
# Saving cross validation results
scipy.io.savemat('xval_results.mat',mdict={'cvconfusion': cvconfusion.tolist()})
return model
def create_mosaic(dataset_in,
clean_mask=None,
no_data=-9999,
dtype=None,
intermediate_product=None,
**kwargs):
"""
Creates a most-recent-to-oldest mosaic of the input dataset.
Parameters
----------
dataset_in: xarray.Dataset
A dataset retrieved from the Data Cube; should contain:
coordinates: time, latitude, longitude
variables: variables to be mosaicked (e.g. red, green, and blue bands)
clean_mask: np.ndarray
An ndarray of the same shape as `dataset_in` - specifying which values to mask out.
If no clean mask is specified, then all values are kept during compositing.
no_data: int or float
The no data value.
dtype: str or numpy.dtype
A string denoting a Python datatype name (e.g. int, float) or a NumPy dtype (e.g.
np.int16, np.float32) to convert the data to.
Returns
-------
dataset_out: xarray.Dataset
Compositited data with the format:
coordinates: latitude, longitude
variables: same as dataset_in
"""
dataset_in = dataset_in.copy(deep=True)
# Default to masking nothing.
if clean_mask is None:
clean_mask = create_default_clean_mask(dataset_in)
# Mask data with clean_mask. All values where clean_mask==False are set to no_data.
for key in list(dataset_in.data_vars):
dataset_in[key].values[np.invert(clean_mask)] = no_data
# Save dtypes because masking with Dataset.where() converts to float64.
band_list = list(dataset_in.data_vars)
dataset_in_dtypes = {}
for band in band_list:
dataset_in_dtypes[band] = dataset_in[band].dtype
if intermediate_product is not None:
dataset_out = intermediate_product.copy(deep=True)
else:
dataset_out = None
time_slices = reversed(range(len(
dataset_in.time))) if 'reverse_time' in kwargs else range(
len(dataset_in.time))
for index in time_slices:
dataset_slice = dataset_in.isel(time=index).drop('time')
if dataset_out is None:
dataset_out = dataset_slice.copy(deep=True)
utilities.clear_attrs(dataset_out)
else:
for key in list(dataset_in.data_vars):
dataset_out[key].values[dataset_out[key].values ==
-9999] = dataset_slice[key].values[
dataset_out[key].values == -9999]
dataset_out[key].attrs = OrderedDict()
# Handle datatype conversions.
dataset_out = restore_or_convert_dtypes(dtype, band_list,
dataset_in_dtypes, dataset_out,
no_data)
return dataset_out
Qs1 = Qs1 + cos(w * (NV - 0.5)) * np.array(
(cos(w * (NV - 0.5)))).transpose()
Qs1 = ws1 * Qs1 / (Ns_s1 + 1)
Qs2 = np.zeros(NH, NH)
for iw in range(Ns_s2):
w = ws2 + iw * deltaw
Qs2 = Qs2 + cos(w * (NV - 0.5)) * np.array(
(cos(w * (NV - 0.5)))).transpose()
Qs2 = (pi - ws2) * Qs2 / (Ns_s2 + 1)
Q = Qp + Qs1 + Qs2
A = -0.5 * np.invert(Q) * P
h = np.zeros(N, 1)
h[1:NH] = 0.5 * np.flipud(A)
h[NH + 1:N] = 0.5 * A
# figure, (ax1, ax2, ax3, ax4) = plt.subplots(2, 2)
# ax1.stem([0:N-1], h)
# plt.stem([0:N-1], h)
plt.stem(range(N), h)
plt.xlabel('n')
plt.ylabel('Impulse response')
def flag_cr(sci_image, blot_image, **pars):
"""Masks outliers in science image.
Mask blemishes in dithered data by comparing a science image
with a model image and the derivative of the model image.
Parameters
----------
sci_image : ImageModel
the science data
blot_image : ImageModel
the blotted median image of the dithered science frames
pars : dict
the user parameters for Outlier Detection
Default parameters:
grow = 1 # Radius to mask [default=1 for 3x3]
ctegrow = 0 # Length of CTE correction to be applied
snr = "5.0 4.0" # Signal-to-noise ratio
scale = "1.2 0.7" # scaling factor applied to the derivative
backg = 0 # Background value
"""
grow = pars.get('grow', 1)
ctegrow = pars.get('ctegrow', 0) # not provided by outlierpars
backg = pars.get('backg', 0)
snr1, snr2 = [float(val) for val in pars.get('snr', '5.0 4.0').split()]
scl1, scl2 = [float(val) for val in pars.get('scale', '1.2 0.7').split()]
if not sci_image.meta.background.subtracted:
# Include background back into blotted image for comparison
subtracted_background = sci_image.meta.background.level
log.debug("Subtracted background: {}".format(subtracted_background))
if subtracted_background is None:
subtracted_background = backg
exptime = sci_image.meta.exposure.exposure_time
sci_data = sci_image.data * exptime
blot_data = blot_image.data * exptime
blot_deriv = abs_deriv(blot_data)
err_data = np.nan_to_num(sci_image.err)
# Define output cosmic ray mask to populate
cr_mask = np.zeros(sci_image.shape, dtype=np.uint8)
#
#
# COMPUTATION PART I
#
#
# Model the noise and create a CR mask
diff_noise = np.abs(sci_data - blot_data)
# ta = np.sqrt(np.abs(blot_data + subtracted_background) + rn ** 2)
ta = np.sqrt(np.abs(blot_data + subtracted_background) + err_data**2)
t2 = scl1 * blot_deriv + snr1 * ta
tmp1 = np.logical_not(np.greater(diff_noise, t2))
# Convolve mask with 3x3 kernel
kernel = np.ones((3, 3), dtype=np.uint8)
tmp2 = np.zeros(tmp1.shape, dtype=np.int32)
ndimage.convolve(tmp1, kernel, output=tmp2, mode='nearest', cval=0)
#
#
# COMPUTATION PART II
#
#
# Create a second CR Mask
xt2 = scl2 * blot_deriv + snr2 * ta
np.logical_not(np.greater(diff_noise, xt2) & np.less(tmp2, 9), cr_mask)
#
#
# COMPUTATION PART III
#
#
# Flag additional cte 'radial' and 'tail' pixels surrounding CR
# pixels as CRs
# In both the 'radial' and 'length' kernels below, 0=good and
# 1=bad, so that upon convolving the kernels with cr_mask, the
# convolution output will have low->bad and high->good from which
# 2 new arrays are created having 0->bad and 1->good. These 2 new
# arrays are then AND'ed to create a new cr_mask.
# recast cr_mask to int for manipulations below; will recast to
# Bool at end
cr_mask_orig_bool = cr_mask.copy()
cr_mask = cr_mask_orig_bool.astype(np.int8)
# make radial convolution kernel and convolve it with original cr_mask
cr_grow_kernel = np.ones((grow, grow))
cr_grow_kernel_conv = cr_mask.copy()
ndimage.convolve(cr_mask, cr_grow_kernel, output=cr_grow_kernel_conv)
# make tail convolution kernel and (shortly) convolve it with
# original cr_mask
cr_ctegrow_kernel = np.zeros((2 * ctegrow + 1, 2 * ctegrow + 1))
cr_ctegrow_kernel_conv = cr_mask.copy()
# which pixels are masked by tail kernel depends on readout direction
# We could put useful info in here for CTE masking if needed. Code
# remains below. For now, we set to zero, which turns off CTE masking.
ctedir = 0
if (ctedir == 1):
cr_ctegrow_kernel[0:ctegrow, ctegrow] = 1
if (ctedir == -1):
cr_ctegrow_kernel[ctegrow + 1:2 * ctegrow + 1, ctegrow] = 1
if (ctedir == 0):
pass
# finally do the tail convolution
ndimage.convolve(cr_mask, cr_ctegrow_kernel, output=cr_ctegrow_kernel_conv)
# select high pixels from both convolution outputs; then 'and' them to
# create new cr_mask
where_cr_grow_kernel_conv = np.where(cr_grow_kernel_conv < grow * grow, 0,
1)
where_cr_ctegrow_kernel_conv = np.where(cr_ctegrow_kernel_conv < ctegrow,
0, 1)
# combine masks and cast back to Bool
np.logical_and(where_cr_ctegrow_kernel_conv, where_cr_grow_kernel_conv,
cr_mask)
cr_mask = cr_mask.astype(bool)
count_sci = np.count_nonzero(sci_image.dq)
count_cr = np.count_nonzero(cr_mask)
log.debug("Pixels in input DQ: {}".format(count_sci))
log.debug("Pixels in cr_mask: {}".format(count_cr))
# Update the DQ array in the input image in place
np.bitwise_or(sci_image.dq, np.invert(cr_mask) * CRBIT, sci_image.dq)
def get_image(filename):
try:
return np.invert(np.array(Image.open(filename).convert("L"))) / 255
except IOError:
print("Error in getting image file: File not available!")
def event_extraction_and_comparison(sr):
# it took 8 min to run that for 6 min of data, all 300 ish channels
# silent channels for Guido's set:
# [36,75,112,151,188,227,264,303,317,340,379,384]
# sr,sync=get_ephys_data(sync_test_folder)
"""
this function first finds the times of square signal fronts in ephys and
compares them to corresponding ones in the sync signal.
Iteratively for small data chunks
"""
_logger.info('starting event_extraction_and_comparison')
period_duration = 30000 # in observations, 30 kHz
BATCH_SIZE_SAMPLES = period_duration # in observations, 30 kHz
# only used to find first pulse
wg = dsp.WindowGenerator(sr.ns, BATCH_SIZE_SAMPLES, overlap=1)
# if the data is needed as well, loop over the file
# raw data contains raw ephys traces, while raw_sync contains the 16 sync
# traces
rawdata, _ = sr.read_samples(0, BATCH_SIZE_SAMPLES)
_, chans = rawdata.shape
chan_fronts = {}
sync_fronts = {}
sync_fronts['fpga up fronts'] = []
sync_fronts['fpga down fronts'] = []
for j in range(chans):
chan_fronts[j] = {}
chan_fronts[j]['ephys up fronts'] = []
chan_fronts[j]['ephys down fronts'] = []
# find time of first pulse (take first channel with square signal)
for i in range(chans):
try:
# assuming a signal in the first minute
for first, last in list(wg.firstlast)[:120]:
rawdata, rawsync = sr.read_samples(first, last)
diffs = np.diff(rawsync.T[0])
sync_up_fronts = np.where(diffs == 1)[0] + first
if len(sync_up_fronts) != 0:
break
assert len(sync_up_fronts) != 0
Channel = i
break
except BaseException:
print('channel %s shows no pulse signal, checking next' % i)
assert i < 10, \
"something wrong, \
the first 10 channels don't show a square signal"
continue
start_of_chopping = sync_up_fronts[0] - period_duration / 4
k = 0
# assure there is exactly one pulse per cut segment
for pulse in range(500): # there are 500 square pulses
first = int(start_of_chopping + period_duration * pulse)
last = int(first + period_duration)
if k % 100 == 0:
print('segment %s of %s' % (k, 500))
k += 1
rawdata, rawsync = sr.read_samples(first, last)
# get fronts for sync signal
diffs = np.diff(rawsync.T[0]) # can that thing be a global variable?
sync_up_fronts = np.where(diffs == 1)[0] + first
sync_down_fronts = np.where(diffs == -1)[0] + first
sync_fronts['fpga up fronts'].append(sync_up_fronts)
sync_fronts['fpga down fronts'].append(sync_down_fronts)
# get fronts for only one valid ephys channel
obs, chans = rawdata.shape
i = Channel
Mean = np.median(rawdata.T[i])
Std = np.std(rawdata.T[i])
ups = np.invert(rawdata.T[i] > Mean + 6 * Std)
downs = np.invert(rawdata.T[i] < Mean - 6 * Std)
up_fronts = []
down_fronts = []
# Activity front at least 10 samples long (empirical)
up_fronts.append(first_occ_index(ups, 20) + first)
down_fronts.append(first_occ_index(downs, 20) + first)
chan_fronts[i]['ephys up fronts'].append(up_fronts)
chan_fronts[i]['ephys down fronts'].append(down_fronts)
return chan_fronts, sync_fronts # all differences
def eval_cuhk03(distmat, q_pids, g_pids, q_camids, g_camids, max_rank, N=100):
"""Evaluation with cuhk03 metric
Key: one image for each gallery identity is randomly sampled for each query identity.
Random sampling is performed N times (default: N=100).
"""
num_q, num_g = distmat.shape
if num_g < max_rank:
max_rank = num_g
print("Note: number of gallery samples is quite small, got {}".format(
num_g))
indices = np.argsort(distmat, axis=1)
# 重要的matches
matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32)
# compute cmc curve for each query
all_cmc = []
all_AP = []
num_valid_q = 0. # number of valid query
for q_idx in range(num_q):
# get query pid and camid
q_pid = q_pids[q_idx]
q_camid = q_camids[q_idx]
# remove gallery samples that have the same pid and camid with query
order = indices[q_idx]
remove = (g_pids[order] == q_pid) & (g_camids[order] == q_camid)
keep = np.invert(remove)
# compute cmc curve
orig_cmc = matches[q_idx][
keep] # binary vector, positions with value 1 are correct matches
# embed()
if not np.any(orig_cmc):
# this condition is true when query identity does not appear in gallery
continue
kept_g_pids = g_pids[order][keep]
g_pids_dict = defaultdict(list)
for idx, pid in enumerate(kept_g_pids):
g_pids_dict[pid].append(idx)
cmc, AP = 0., 0.
for repeat_idx in range(N):
mask = np.zeros(len(orig_cmc), dtype=np.bool)
for _, idxs in g_pids_dict.items():
# randomly sample one image for each gallery person
rnd_idx = np.random.choice(idxs)
mask[rnd_idx] = True
masked_orig_cmc = orig_cmc[mask]
_cmc = masked_orig_cmc.cumsum()
_cmc[_cmc > 1] = 1
cmc += _cmc[:max_rank].astype(np.float32)
# compute AP
num_rel = masked_orig_cmc.sum()
tmp_cmc = masked_orig_cmc.cumsum()
tmp_cmc = [x / (i + 1.) for i, x in enumerate(tmp_cmc)]
tmp_cmc = np.asarray(tmp_cmc) * masked_orig_cmc
AP += tmp_cmc.sum() / num_rel
cmc /= N
AP /= N
all_cmc.append(cmc)
all_AP.append(AP)
num_valid_q += 1.
assert num_valid_q > 0, "Error: all query identities do not appear in gallery"
all_cmc = np.asarray(all_cmc).astype(np.float32)
all_cmc = all_cmc.sum(0) / num_valid_q
mAP = np.mean(all_AP)
return all_cmc, mAP
def ON_bayesian_constant(device, duration, thresh):
# device = matrix which is output of device_status_matrix.py aka ON OFF Missing data matrix
# duration = time duration in minutes for state to be decided wether ON OFF
# thresh = threshold to PMF to decide device is ON or OFF
################# Time axis #####################
time_min = np.arange(1, device.shape[-1] + 1) * duration
mask = device.copy()
mask[device > -1] = 0
mask = np.abs(mask)
#### days were whole day data collected #######
active = np.invert(mask.any(axis=1))
device = device[active, :]
# np.random.shuffle(device)
ind = int(len(device) * 0.3)
dev_test_org = device[-ind:, :] #.flatten() # test data
previous_prior = np.sum(device[:ind, :], axis=0,
dtype='float32') #1 # constant prior
previous_prior = previous_prior / previous_prior.sum()
dev_test_pred = []
Acc = []
F1Score = []
Preci = []
for itr in range(ind):
likely_hood = np.sum(device[ind:-ind + itr, :],
axis=0,
dtype='float32')
#likely_hood = likely_hood/likely_hood.sum()
dev_test_day = likely_hood * previous_prior
dev_test_day[dev_test_day >= thresh] = 1
dev_test_day[dev_test_day < thresh] = 0
dev_test_pred.append(dev_test_day)
Acc.append(round(accuracy_score(dev_test_org[itr, :], dev_test_day),
4))
F1Score.append(round(f1_score(dev_test_org[itr, :], dev_test_day), 4))
Preci.append(
round(precision_score(dev_test_org[itr, :], dev_test_day), 4))
dev_test_pred = np.array(dev_test_pred, np.float32).flatten()
conf_mat = confusion_matrix(dev_test_org.flatten(), dev_test_pred)
CM_Acc_F1_Prec = [
conf_mat,
Acc,
F1Score,
Preci,
]
# this will give day wise what time data collected
"""
observed_data = device.copy()
observed_data[observed_data > -1] = 1
observed_data[observed_data < 1] = 0
plt_obser = np.sum(observed_data,axis=0)
# this will give day wise what time device is ON
on_data = device.copy()
on_data[on_data < 1] = 0
plt_on = np.sum(on_data,axis=0)
"""
return time_min, previous_prior, CM_Acc_F1_Prec
def _write_poisson_parameters(self, spec, graph, placement, routing_info,
vertex_slice, machine_time_step,
time_scale_factor):
""" Generate Neuron Parameter data for Poisson spike sources
:param spec: the data specification writer
:param key: the routing key for this vertex
:param vertex_slice:\
the slice of atoms a machine vertex holds from its application\
vertex
:param machine_time_step: the time between timer tick updates.
:param time_scale_factor:\
the scaling between machine time step and real time
:return: None
"""
# pylint: disable=too-many-arguments, too-many-locals
spec.comment(
"\nWriting Neuron Parameters for {} poisson sources:\n".format(
vertex_slice.n_atoms))
# Set the focus to the memory region 2 (neuron parameters):
spec.switch_write_focus(_REGIONS.POISSON_PARAMS_REGION.value)
# Write Key info for this core:
key = routing_info.get_first_key_from_pre_vertex(
placement.vertex, constants.SPIKE_PARTITION_ID)
spec.write_value(data=1 if key is not None else 0)
spec.write_value(data=key if key is not None else 0)
# Write the incoming mask if there is one
in_edges = graph.get_edges_ending_at_vertex_with_partition_name(
placement.vertex, constants.LIVE_POISSON_CONTROL_PARTITION_ID)
if len(in_edges) > 1:
raise ConfigurationException(
"Only one control edge can end at a Poisson vertex")
incoming_mask = 0
if len(in_edges) == 1:
in_edge = in_edges[0]
# Get the mask of the incoming keys
incoming_mask = \
routing_info.get_routing_info_for_edge(in_edge).first_mask
incoming_mask = ~incoming_mask & 0xFFFFFFFF
spec.write_value(incoming_mask)
# Write the offset value
max_offset = (machine_time_step *
time_scale_factor) // _MAX_OFFSET_DENOMINATOR
spec.write_value(
int(math.ceil(max_offset / self.__n_subvertices)) *
self.__n_data_specs)
self.__n_data_specs += 1
# Write the number of microseconds between sending spikes
total_mean_rate = numpy.sum(self.__rate)
if total_mean_rate > 0:
max_spikes = numpy.sum(
scipy.stats.poisson.ppf(1.0 - (1.0 / self.__rate),
self.__rate))
spikes_per_timestep = (
max_spikes / (MICROSECONDS_PER_SECOND // machine_time_step))
# avoid a possible division by zero / small number (which may
# result in a value that doesn't fit in a uint32) by only
# setting time_between_spikes if spikes_per_timestep is > 1
time_between_spikes = 1.0
if spikes_per_timestep > 1:
time_between_spikes = (
(machine_time_step * time_scale_factor) /
(spikes_per_timestep * 2.0))
spec.write_value(data=int(time_between_spikes))
else:
# If the rate is 0 or less, set a "time between spikes" of 1
# to ensure that some time is put between spikes in event
# of a rate change later on
spec.write_value(data=1)
# Write the number of seconds per timestep (unsigned long fract)
spec.write_value(data=float(machine_time_step) /
MICROSECONDS_PER_SECOND,
data_type=DataType.U032)
# Write the number of timesteps per second (accum)
spec.write_value(data=MICROSECONDS_PER_SECOND /
float(machine_time_step),
data_type=DataType.S1615)
# Write the slow-rate-per-tick-cutoff (accum)
spec.write_value(data=SLOW_RATE_PER_TICK_CUTOFF,
data_type=DataType.S1615)
# Write the lo_atom id
spec.write_value(data=vertex_slice.lo_atom)
# Write the number of sources
spec.write_value(data=vertex_slice.n_atoms)
# Write the random seed (4 words), generated randomly!
kiss_key = (vertex_slice.lo_atom, vertex_slice.hi_atom)
if kiss_key not in self.__kiss_seed:
if self.__rng is None:
self.__rng = numpy.random.RandomState(self.__seed)
self.__kiss_seed[kiss_key] = [
self.__rng.randint(-0x80000000, 0x7FFFFFFF) + 0x80000000
for _ in range(4)
]
for value in self.__kiss_seed[kiss_key]:
spec.write_value(data=value)
# Compute the start times in machine time steps
start = self.__start[vertex_slice.as_slice]
start_scaled = self._convert_ms_to_n_timesteps(start,
machine_time_step)
# Compute the end times as start times + duration in machine time steps
# (where duration is not None)
duration = self.__duration[vertex_slice.as_slice]
end_scaled = numpy.zeros(len(duration), dtype="uint32")
none_positions = numpy.isnan(duration)
positions = numpy.invert(none_positions)
end_scaled[none_positions] = 0xFFFFFFFF
end_scaled[positions] = self._convert_ms_to_n_timesteps(
start[positions] + duration[positions], machine_time_step)
# Get the rates for the atoms
rates = self.__rate[vertex_slice.as_slice].astype("float")
# Compute the spikes per tick for each atom
spikes_per_tick = (
rates * (float(machine_time_step) / MICROSECONDS_PER_SECOND))
# Determine which sources are fast and which are slow
is_fast_source = spikes_per_tick > SLOW_RATE_PER_TICK_CUTOFF
# Compute the e^-(spikes_per_tick) for fast sources to allow fast
# computation of the Poisson distribution to get the number of spikes
# per timestep
exp_minus_lambda = numpy.zeros(len(spikes_per_tick), dtype="float")
exp_minus_lambda[is_fast_source] = numpy.exp(
-1.0 * spikes_per_tick[is_fast_source])
# Compute the inter-spike-interval for slow sources to get the average
# number of timesteps between spikes
isi_val = numpy.zeros(len(spikes_per_tick), dtype="float")
elements = numpy.logical_not(is_fast_source) & (spikes_per_tick > 0)
isi_val[elements] = 1.0 / spikes_per_tick[elements]
# Get the time to spike value
time_to_spike = self.__time_to_spike[vertex_slice.as_slice]
changed_rates = (
self.__rate_change[vertex_slice.as_slice].astype("bool")
& elements)
time_to_spike[changed_rates] = 0.0
# Merge the arrays as parameters per atom
data = numpy.dstack(
(start_scaled.astype("uint32"), end_scaled.astype("uint32"),
is_fast_source.astype("uint32"),
(exp_minus_lambda * (2**32)).astype("uint32"),
(isi_val * (2**15)).astype("uint32"),
(time_to_spike * (2**15)).astype("uint32")))[0]
spec.write_array(data)
def eval_market1501(distmat, q_pids, g_pids, q_camids, g_camids, max_rank):
"""Evaluation with market1501 metric
Key: for each query identity, its gallery images from the same camera view（来自同一个镜头视角视角的）are discarded.
"""
print('------start eval_market1501------')
num_q, num_g = distmat.shape
if num_g < max_rank: # max_rank=50
max_rank = num_g
print("Note: number of gallery samples is quite small, got {}".format(
num_g))
# 做排序，对相似度做排序
indices = np.argsort(distmat, axis=1)
# 将query集中的每一张图像与gallery集中的图像进行匹配，属于同一id的标为1，换言之，找出pid相同的query和gallery图像
matches = (g_pids[indices] == q_pids[:, np.newaxis]).astype(np.int32)
# embed()
# 打个断点，测试一下
# embed()
# compute cmc curve for each query
all_cmc = []
all_AP = []
num_valid_q = 0. # number of valid query，有效的query数据，这个怎么理解？
for q_idx in range(num_q):
# get query pid and camid
q_pid = q_pids[q_idx]
q_camid = q_camids[q_idx]
# remove gallery samples that have the same pid and camid with query（除掉gallery集中与query图像同时有相同的pid和camid的图像）
order = indices[q_idx]
remove = (g_pids[order] == q_pid) & (g_camids[order] == q_camid)
keep = np.invert(remove) # np.invert()按位NOT
# compute cmc curve
orig_cmc = matches[q_idx][
keep] # binary vector, positions with value 1 are correct matches
# embed()
print(orig_cmc)
# if not np.any(orig_cmc):
# # this condition is true when query identity does not appear in gallery
# continue
# embed()
cmc = orig_cmc.cumsum()
# embed()
cmc[cmc > 1] = 1
all_cmc.append(cmc[:max_rank])
num_valid_q += 1.
# embed()
# compute average precision
# reference: https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#Average_precision
num_rel = orig_cmc.sum()
tmp_cmc = orig_cmc.cumsum()
tmp_cmc = [x / (i + 1.) for i, x in enumerate(tmp_cmc)]
tmp_cmc = np.asarray(tmp_cmc) * orig_cmc
AP = tmp_cmc.sum() / num_rel
# 检查num_rel是否为0值
# embed()
all_AP.append(AP)
assert num_valid_q > 0, "Error: all query identities do not appear in gallery"
# 查看一下all_cmc前后的shape
# embed()
all_cmc = np.asarray(all_cmc).astype(np.float32)
all_cmc = all_cmc.sum(0) / num_valid_q
# embed()
mAP = np.mean(all_AP)
# embed()
print('all_cmc', all_cmc)
print('mAp', mAP)
print('------end eval_market1501------')
return all_cmc, mAP
def write_scrip(self, filename, mask=None, write_test_scrip=True, history=''):
"""
Write out grid in SCRIP format.
"""
self.make_corners()
f = nc.Dataset(filename, 'w')
x = self.x_t
y = self.y_t
clat = self.clat_t
clon = self.clon_t
num_points = self.num_lat_points * self.num_lon_points
f.createDimension('grid_size', num_points)
f.createDimension('grid_corners', 4)
f.createDimension('grid_rank', 2)
grid_dims = f.createVariable('grid_dims', 'i4', ('grid_rank'))
# SCRIP likes lon, lat
grid_dims[:] = [self.num_lon_points, self.num_lat_points]
center_lat = f.createVariable('grid_center_lat', 'f8', ('grid_size'))
center_lat.units = 'degrees'
center_lat[:] = y[:].flatten()
center_lon = f.createVariable('grid_center_lon', 'f8', ('grid_size'))
center_lon.units = 'degrees'
center_lon[:] = x[:].flatten()
imask = f.createVariable('grid_imask', 'i4', ('grid_size'))
imask.units = 'unitless'
# Invert the mask. SCRIP uses zero for points that do not
# participate.
if mask is not None:
mask = mask
else:
mask = self.mask
if len(mask.shape) == 2:
imask[:] = np.invert(mask[:]).flatten()
else:
imask[:] = np.invert(mask[0, :, :]).flatten()
corner_lat = f.createVariable('grid_corner_lat', 'f8',
('grid_size', 'grid_corners'))
corner_lat.units = 'degrees'
corner_lat[:] = clat[:].flatten()
corner_lon = f.createVariable('grid_corner_lon', 'f8',
('grid_size', 'grid_corners'))
corner_lon.units = 'degrees'
corner_lon[:] = clon[:].flatten()
f.title = self.description
f.history = history
f.close()
if write_test_scrip:
self.write_test_scrip(filename + '_test')
import numpy as np
from skimage.transform import resize
from skimage import measure
from skimage.measure import regionprops
import matplotlib.patches as patches
import matplotlib.pyplot as plt
import platedetection
license_plate = np.invert(platedetection.plate_similar[0])
labelled_plate = measure.label(license_plate)
fig, ax1 = plt.subplots(1)
ax1.imshow(license_plate, cmap="gray")
char_dims = (0.35*license_plate.shape[0], 0.60*license_plate.shape[0], 0.05*license_plate.shape[1], 0.15*license_plate.shape[1])
min_height, max_height, min_width, max_width = char_dims
characters = []
counter=0
column_list = []
for regions in regionprops(labelled_plate):
y0, x0, y1, x1 = regions.bbox
height = y1 - y0
width = x1 - x0
if height > min_height and height < max_height and width > min_width and width < max_width:
roi = license_plate[y0:y1, x0:x1]
border = patches.Rectangle((x0, y0), x1 - x0, y1 - y0, edgecolor="blue",
linewidth=2, fill=False)
ax1.add_patch(border)
def refresh_tids(self, tids):
""" Sync documents with `['tid']` in the list of `tids` from the
database (not *to* the database).
Local trial documents whose tid is not in `tids` are not
affected by this call. Local trial documents whose tid is in `tids` may
be:
* *deleted* (if db no longer has corresponding document), or
* *updated* (if db has an updated document) or,
* *left alone* (if db document matches local one).
Additionally, if the db has a matching document, but there is no
local trial with a matching tid, then the db document will be
*inserted* into the local collection.
"""
exp_key = self._exp_key
if exp_key != None:
query = {"exp_key": exp_key}
else:
query = {}
t0 = time.time()
query["state"] = {"$ne": JOB_STATE_ERROR}
if tids is not None:
query["tid"] = {"$in": list(tids)}
orig_trials = getattr(self, "_trials", [])
_trials = orig_trials[:] # copy to make sure it doesn't get screwed up
if _trials:
db_data = list(self.handle.jobs.find(query, projection=["_id", "version"]))
# -- pull down a fresh list of ids from mongo
if db_data:
# make numpy data arrays
db_data = numpy.rec.array(
[(x["_id"], int(x["version"])) for x in db_data],
names=["_id", "version"],
)
db_data.sort(order=["_id", "version"])
db_data = db_data[get_most_recent_inds(db_data)]
existing_data = numpy.rec.array(
[(x["_id"], int(x["version"])) for x in _trials],
names=["_id", "version"],
)
existing_data.sort(order=["_id", "version"])
# which records are in db but not in existing, and vice versa
db_in_existing = fast_isin(db_data["_id"], existing_data["_id"])
existing_in_db = fast_isin(existing_data["_id"], db_data["_id"])
# filtering out out-of-date records
_trials = [_trials[_ind] for _ind in existing_in_db.nonzero()[0]]
# new data is what's in db that's not in existing
new_data = db_data[numpy.invert(db_in_existing)]
# having removed the new and out of data data,
# concentrating on data in db and existing for state changes
db_data = db_data[db_in_existing]
existing_data = existing_data[existing_in_db]
try:
assert len(db_data) == len(existing_data)
assert (existing_data["_id"] == db_data["_id"]).all()
assert (existing_data["version"] <= db_data["version"]).all()
except:
report_path = os.path.join(
os.getcwd(),
"hyperopt_refresh_crash_report_"
+ str(numpy.random.randint(1e8))
+ ".pkl",
)
logger.error(
"HYPEROPT REFRESH ERROR: writing error file to %s" % report_path
)
_file = open(report_path, "w")
pickler.dump(
{"db_data": db_data, "existing_data": existing_data}, _file
)
_file.close()
raise
same_version = existing_data["version"] == db_data["version"]
_trials = [_trials[_ind] for _ind in same_version.nonzero()[0]]
version_changes = existing_data[numpy.invert(same_version)]
# actually get the updated records
update_ids = new_data["_id"].tolist() + version_changes["_id"].tolist()
num_new = len(update_ids)
update_query = copy.deepcopy(query)
update_query["_id"] = {"$in": update_ids}
updated_trials = list(self.handle.jobs.find(update_query))
_trials.extend(updated_trials)
else:
num_new = 0
_trials = []
else:
# this case is for performance, though should be able to be removed
# without breaking correctness.
_trials = list(self.handle.jobs.find(query))
if _trials:
_trials = [_trials[_i] for _i in get_most_recent_inds(_trials)]
num_new = len(_trials)
logger.debug(
"Refresh data download took %f seconds for %d ids"
% (time.time() - t0, num_new)
)
if tids is not None:
# -- If tids were given, then _trials only contains
# documents with matching tids. Here we augment these
# fresh matching documents, with our current ones whose
# tids don't match.
new_trials = _trials
tids_set = set(tids)
assert all(t["tid"] in tids_set for t in new_trials)
old_trials = [t for t in orig_trials if t["tid"] not in tids_set]
_trials = new_trials + old_trials
# -- reassign new trials to self, in order of increasing tid
jarray = numpy.array([j["_id"] for j in _trials])
jobsort = jarray.argsort()
self._trials = [_trials[_idx] for _idx in jobsort]
self._specs = [_trials[_idx]["spec"] for _idx in jobsort]
self._results = [_trials[_idx]["result"] for _idx in jobsort]
self._miscs = [_trials[_idx]["misc"] for _idx in jobsort]
def extract_column_from_tables(self,
shape,
sky_crds,
filestr,
basedir=".",
**kwargs):
"""
Extract data from files built as an roi_set
The data are assumed to be in a set of files with names like
roi_00000/tsmap.fits
Parameters
----------
sky_crds : array-like (2,ndir)
Directions to extract data for
filestr : str
Name of file within each ROI sub-directory
basedir : str
Path to top-level directory
tabname : str
Name of table containing data to extract
colname : std
Name of column to extract
Returns
-------
out_array : ~numpy.ndarray(shape)
Array with the extracted data
"""
nbin = kwargs.get('nbin', None)
# Mask out the stuff outide the projections
mask = np.invert(np.isnan(sky_crds)).any(1)
good_sky = sky_crds[mask]
if nbin is not None:
full_mask = np.expand_dims(mask, -1) * np.ones((nbin), bool)
full_mask = full_mask.reshape((full_mask.size))
else:
full_mask = mask
# This is the array we are filling
out_array = np.zeros(shape)
out_array.flat[np.invert(full_mask.flat)] = np.nan
# Figure out which ROI each direction belongs to
rois = self.lb_to_idx(good_sky[0:, 0], good_sky[0:, 1])
# This is an array of only the pixels inside the projection
if nbin is None:
copy_array = np.zeros((mask.sum()))
else:
copy_array = np.zeros((mask.sum(), nbin))
# Loop over ROIs
min_roi = rois.min()
max_roi = rois.max()
for iroi in range(min_roi, max_roi + 1):
# progress
if iroi % 10 == 0:
sys.stdout.write('.')
sys.stdout.flush()
# Mask out the directions for this ROI
local_mask = rois == iroi
if not local_mask.any():
continue
# Open the corresonding file
filepath = os.path.join(basedir, "%s%05i" % (self.prefix, iroi),
filestr)
# Extract the data for this ROI
copy_array[local_mask] = extract_column_data_from_file(
filepath, good_sky[0:, 0][local_mask],
good_sky[0:, 1][local_mask], **kwargs)
out_array.flat[full_mask] = copy_array.flat
return out_array
def _gradient_descent(objective,
p0,
it,
n_iter,
objective_error=None,
n_iter_check=1,
n_iter_without_progress=50,
momentum=0.5,
learning_rate=1000.0,
min_gain=0.01,
min_grad_norm=1e-7,
min_error_diff=1e-7,
verbose=0,
args=None,
kwargs=None):
"""Batch gradient descent with momentum and individual gains.
Parameters
----------
objective : function or callable
Should return a tuple of cost and gradient for a given parameter
vector. When expensive to compute, the cost can optionally
be None and can be computed every n_iter_check steps using
the objective_error function.
p0 : array-like, shape (n_params,)
Initial parameter vector.
it : int
Current number of iterations (this function will be called more than
once during the optimization).
n_iter : int
Maximum number of gradient descent iterations.
n_iter_check : int
Number of iterations before evaluating the global error. If the error
is sufficiently low, we abort the optimization.
objective_error : function or callable
Should return a tuple of cost and gradient for a given parameter
vector.
n_iter_without_progress : int, optional (default: 30)
Maximum number of iterations without progress before we abort the
optimization.
momentum : float, within (0.0, 1.0), optional (default: 0.5)
The momentum generates a weight for previous gradients that decays
exponentially.
learning_rate : float, optional (default: 1000.0)
The learning rate should be extremely high for t-SNE! Values in the
range [100.0, 1000.0] are common.
min_gain : float, optional (default: 0.01)
Minimum individual gain for each parameter.
min_grad_norm : float, optional (default: 1e-7)
If the gradient norm is below this threshold, the optimization will
be aborted.
min_error_diff : float, optional (default: 1e-7)
If the absolute difference of two successive cost function values
is below this threshold, the optimization will be aborted.
verbose : int, optional (default: 0)
Verbosity level.
args : sequence
Arguments to pass to objective function.
kwargs : dict
Keyword arguments to pass to objective function.
Returns
-------
p : array, shape (n_params,)
Optimum parameters.
error : float
Optimum.
i : int
Last iteration.
"""
if args is None:
args = []
if kwargs is None:
kwargs = {}
p = p0.copy().ravel()
update = np.zeros_like(p)
gains = np.ones_like(p)
error = np.finfo(np.float).max
best_error = np.finfo(np.float).max
best_iter = 0
for i in range(it, n_iter):
new_error, grad = objective(p, *args, **kwargs)
grad_norm = linalg.norm(grad)
inc = update * grad >= 0.0
dec = np.invert(inc)
gains[inc] += 0.05
gains[dec] *= 0.95
np.clip(gains, min_gain, np.inf)
grad *= gains
update = momentum * update - learning_rate * grad
p += update
if (i + 1) % n_iter_check == 0:
if new_error is None:
new_error = objective_error(p, *args)
error_diff = np.abs(new_error - error)
error = new_error
if verbose >= 2:
m = "[t-SNE] Iteration %d: error = %.7f, gradient norm = %.7f"
print(m % (i + 1, error, grad_norm))
if error < best_error:
best_error = error
best_iter = i
elif i - best_iter > n_iter_without_progress:
if verbose >= 2:
print("[t-SNE] Iteration %d: did not make any progress "
"during the last %d episodes. Finished." %
(i + 1, n_iter_without_progress))
break
if grad_norm <= min_grad_norm:
if verbose >= 2:
print("[t-SNE] Iteration %d: gradient norm %f. Finished." %
(i + 1, grad_norm))
break
if error_diff <= min_error_diff:
if verbose >= 2:
m = "[t-SNE] Iteration %d: error difference %f. Finished."
print(m % (i + 1, error_diff))
break
if new_error is not None:
error = new_error
return p, error, i
batch_x, batch_y = mnist.train.next_batch(batch_size)
sess.run(train_step,
feed_dict={
X: batch_x,
Y: batch_y,
keep_prob: dropout
})
# print loss and accuracy (per minibatch)
if i % 100 == 0:
minibatch_loss, minibatch_accuracy = sess.run(
[cross_entropy, accuracy],
feed_dict={
X: batch_x,
Y: batch_y,
keep_prob: 1.0
})
print("Iteration", str(i), "\t| Loss =", str(minibatch_loss),
"\t| Accuracy =", str(minibatch_accuracy))
test_accuracy = sess.run(accuracy,
feed_dict={
X: mnist.test.images,
Y: mnist.test.labels,
keep_prob: 1.0
})
print("\nAccuracy on test set:", test_accuracy)
img = np.invert(Image.open("test_img.png").convert('L')).ravel()
prediction = sess.run(tf.argmax(output_layer, 1), feed_dict={X: [img]})
print("Prediction for test image:", np.squeeze(prediction))
def _check_done(self):
"""
Check if the episode is done
This is in terms of the ego car
For our case, whether the car ends up close enough to the starting point
And if accumulated time is over the timeout
return true if done, false if not
This assumes start is always (0, 0)
"""
# TODO: start not always 0, 0
# dist_to_start = math.sqrt((self.x-self.start_x) ** 2 + (self.y-self.start_y) ** 2)
left_t = 1.5
right_t = 5
timeout = self.current_time >= self.timeout
if self.double_finish:
poses_x = np.array(self.all_x)-self.start_xs
poses_y = np.array(self.all_y)-self.start_ys
delta_pt = np.dot(self.start_rot, np.stack((poses_x, poses_y), axis=0))
temp_y = delta_pt[1,:]
idx1 = temp_y > left_t
idx2 = temp_y < -right_t
temp_y[idx1] -= left_t
temp_y[idx2] = -right_t - temp_y[idx2]
temp_y[np.invert(np.logical_or(idx1, idx2))] = 0
dist2 = delta_pt[0,:]**2 + temp_y**2
closes = dist2 <= 0.1
for i in range(self.num_agents):
if closes[i] and not self.near_starts[i]:
self.near_starts[i] = True
self.toggle_list[i] += 1
elif not closes[i] and self.near_starts[i]:
self.near_starts[i] = False
self.toggle_list[i] += 1
done = (self.in_collision | (timeout) | np.all(self.toggle_list >= 4))
# only for two cars atm
self.lap_counts[0] = np.floor(self.toggle_list[0] / 2)
self.lap_counts[1] = np.floor(self.toggle_list[1] / 2)
if self.toggle_list[0] < 4:
self.lap_times[0] = self.current_time
if self.toggle_list[1] < 4:
self.lap_times[1] = self.current_time
return done, self.toggle_list >= 4
delta_pt = np.dot(self.start_rot, np.array([self.x-self.start_x, self.y-self.start_y]))
if delta_pt[1] > left_t: # left
temp_y = delta_pt[1]-left_t
elif delta_pt[1] < -right_t: # right
temp_y = -right_t - delta_pt[1]
else:
temp_y = 0
dist2 = delta_pt[0]**2 + temp_y**2
close = dist2 <= 0.1
# close = dist_to_start <= self.start_thresh
if close and not self.near_start:
self.near_start = True
self.num_toggles += 1
elif not close and self.near_start:
self.near_start = False
self.num_toggles += 1
done = (self.in_collision | (timeout) | (self.num_toggles >= 4))
return done
def calculate_antenna_params(filename,
tltxnumber=1,
tlrxnumber=None,
rxnumber=None,
rxcomponent=None):
"""Calculates antenna parameters - incident, reflected and total volatges and currents; s11, (s21) and input impedance.
Args:
filename (string): Filename (including path) of output file.
tltxnumber (int): Transmitter antenna - transmission line number
tlrxnumber (int): Receiver antenna - transmission line number
rxnumber (int): Receiver antenna - output number
rxcomponent (str): Receiver antenna - output electric field component
Returns:
antennaparams (dict): Antenna parameters.
"""
# Open output file and read some attributes
f = h5py.File(filename, 'r')
dxdydz = f.attrs['dx, dy, dz']
dt = f.attrs['dt']
iterations = f.attrs['Iterations']
# Calculate time array and frequency bin spacing
time = np.linspace(0, 1, iterations)
time *= (iterations * dt)
df = 1 / np.amax(time)
print('Time window: {:g} s ({} iterations)'.format(np.amax(time),
iterations))
print('Time step: {:g} s'.format(dt))
print('Frequency bin spacing: {:g} Hz'.format(df))
# Read/calculate voltages and currents from transmitter antenna
tltxpath = '/tls/tl' + str(tltxnumber) + '/'
# Incident voltages/currents
Vinc = f[tltxpath + 'Vinc'][:]
Iinc = f[tltxpath + 'Iinc'][:]
# Total (incident + reflected) voltages/currents
Vtotal = f[tltxpath + 'Vtotal'][:]
Itotal = f[tltxpath + 'Itotal'][:]
# Reflected voltages/currents
Vref = Vtotal - Vinc
Iref = Itotal - Iinc
# If a receiver antenna is used (with a transmission line or receiver), get received voltage for s21
if tlrxnumber:
tlrxpath = '/tls/tl' + str(tlrxnumber) + '/'
Vrec = f[tlrxpath + 'Vtotal'][:]
elif rxnumber:
rxpath = '/rxs/rx' + str(rxnumber) + '/'
availableoutputs = list(f[rxpath].keys())
if rxcomponent not in availableoutputs:
raise CmdInputError(
'{} output requested, but the available output for receiver {} is {}'
.format(rxcomponent, rxnumber, ', '.join(availableoutputs)))
rxpath += rxcomponent
# Received voltage
if rxcomponent == 'Ex':
Vrec = f[rxpath][:] * -1 * dxdydz[0]
elif rxcomponent == 'Ey':
Vrec = f[rxpath][:] * -1 * dxdydz[1]
elif rxcomponent == 'Ez':
Vrec = f[rxpath][:] * -1 * dxdydz[2]
f.close()
# Frequency bins
freqs = np.fft.fftfreq(Vinc.size, d=dt)
# Delay correction - current lags voltage, so delay voltage to match current timestep
delaycorrection = np.exp(1j * 2 * np.pi * freqs * (dt / 2))
# Calculate s11 and (optionally) s21
s11 = np.abs(np.fft.fft(Vref) / np.fft.fft(Vinc))
if tlrxnumber or rxnumber:
s21 = np.abs(np.fft.fft(Vrec) / np.fft.fft(Vinc))
# Calculate input impedance
zin = (np.fft.fft(Vtotal) * delaycorrection) / np.fft.fft(Itotal)
# Calculate input admittance
yin = np.fft.fft(Itotal) / (np.fft.fft(Vtotal) * delaycorrection)
# Convert to decibels (ignore warning from taking a log of any zero values)
with np.errstate(divide='ignore'):
Vincp = 20 * np.log10(np.abs((np.fft.fft(Vinc) * delaycorrection)))
Iincp = 20 * np.log10(np.abs(np.fft.fft(Iinc)))
Vrefp = 20 * np.log10(np.abs((np.fft.fft(Vref) * delaycorrection)))
Irefp = 20 * np.log10(np.abs(np.fft.fft(Iref)))
Vtotalp = 20 * np.log10(np.abs((np.fft.fft(Vtotal) * delaycorrection)))
Itotalp = 20 * np.log10(np.abs(np.fft.fft(Itotal)))
s11 = 20 * np.log10(s11)
# Replace any NaNs or Infs from zero division
Vincp[np.invert(np.isfinite(Vincp))] = 0
Iincp[np.invert(np.isfinite(Iincp))] = 0
Vrefp[np.invert(np.isfinite(Vrefp))] = 0
Irefp[np.invert(np.isfinite(Irefp))] = 0
Vtotalp[np.invert(np.isfinite(Vtotalp))] = 0
Itotalp[np.invert(np.isfinite(Itotalp))] = 0
s11[np.invert(np.isfinite(s11))] = 0
# Create dictionary of antenna parameters
antennaparams = {
'time': time,
'freqs': freqs,
'Vinc': Vinc,
'Vincp': Vincp,
'Iinc': Iinc,
'Iincp': Iincp,
'Vref': Vref,
'Vrefp': Vrefp,
'Iref': Iref,
'Irefp': Irefp,
'Vtotal': Vtotal,
'Vtotalp': Vtotalp,
'Itotal': Itotal,
'Itotalp': Itotalp,
's11': s11,
'zin': zin,
'yin': yin
}
if tlrxnumber or rxnumber:
with np.errstate(
divide='ignore'
): # Ignore warning from taking a log of any zero values
s21 = 20 * np.log10(s21)
s21[np.invert(np.isfinite(s21))] = 0
antennaparams['s21'] = s21
return antennaparams
