def evaluate(gtdir, preddir, eval_pose=True, eval_track=True,
eval_upper_bound=False):
gtFramesAll, prFramesAll = load_data_dir(['', gtdir, preddir])
print('# gt frames :', len(gtFramesAll))
print('# pred frames:', len(prFramesAll))
apAll = np.full((Joint().count + 1, 1), np.nan)
preAll = np.full((Joint().count + 1, 1), np.nan)
recAll = np.full((Joint().count + 1, 1), np.nan)
if eval_pose:
apAll, preAll, recAll = evaluateAP(gtFramesAll, prFramesAll)
print('Average Precision (AP) metric:')
printTable(apAll)
metrics = np.full((Joint().count + 4, 1), np.nan)
if eval_track:
metricsAll = evaluateTracking(
gtFramesAll, prFramesAll, eval_upper_bound)
for i in range(Joint().count + 1):
metrics[i, 0] = metricsAll['mota'][0, i]
metrics[Joint().count + 1, 0] = metricsAll['motp'][0, Joint().count]
metrics[Joint().count + 2, 0] = metricsAll['pre'][0, Joint().count]
metrics[Joint().count + 3, 0] = metricsAll['rec'][0, Joint().count]
print('Multiple Object Tracking (MOT) metrics:')
printTable(metrics, motHeader=True)
return (apAll, preAll, recAll), metrics
def testSequenceLoss(self):
self.config_default_values()
with self.cached_session(use_gpu=True):
average_loss_per_example = loss.sequence_loss(
self.logits, self.targets, self.weights,
average_across_timesteps=True,
average_across_batch=True)
res = self.evaluate(average_loss_per_example)
self.assertAllClose(self.expected_loss, res)
average_loss_per_sequence = loss.sequence_loss(
self.logits, self.targets, self.weights,
average_across_timesteps=False,
average_across_batch=True)
res = self.evaluate(average_loss_per_sequence)
compare_per_sequence = np.full((self.sequence_length), self.expected_loss)
self.assertAllClose(compare_per_sequence, res)
average_loss_per_batch = loss.sequence_loss(
self.logits, self.targets, self.weights,
average_across_timesteps=True,
average_across_batch=False)
res = self.evaluate(average_loss_per_batch)
compare_per_batch = np.full((self.batch_size), self.expected_loss)
self.assertAllClose(compare_per_batch, res)
total_loss = loss.sequence_loss(
self.logits, self.targets, self.weights,
average_across_timesteps=False,
average_across_batch=False)
res = self.evaluate(total_loss)
compare_total = np.full((self.batch_size, self.sequence_length),
self.expected_loss)
self.assertAllClose(compare_total, res)
def at(self, points, time, units=None, extrapolate=False, **kwargs):
'''
Interpolates this property to the given points at the given time with the units specified
:param points: A Nx2 array of lon,lat points
:param time: A datetime object. May be None; if this is so, the variable is assumed to be gridded
but time-invariant
:param units: The units that the result would be converted to
'''
value = None
if len(self.time) == 1:
# single time time series (constant)
value = np.full((points.shape[0], 1), self.data, dtype=np.float64)
if units is not None and units != self.units:
value = unit_conversion.convert(self.units, units, value)
return value
if not extrapolate:
self.time.valid_time(time)
t_index = self.time.index_of(time, extrapolate)
if time > self.time.max_time:
value = self.data[-1]
if time <= self.time.min_time:
value = self.data[0]
if value is None:
t_alphas = self.time.interp_alpha(time, extrapolate)
d0 = self.data[t_index - 1]
d1 = self.data[t_index]
value = d0 + (d1 - d0) * t_alphas
if units is not None and units != self.units:
value = unit_conversion.convert(self.units, units, value)
return np.full((points.shape[0], 1), value, dtype=np.float64)
def _clean_timeseries(self, timeseries, starttime, endtime):
"""Realigns timeseries data so the start and endtimes are the same
as what was originally asked for, even if the data was during
a gap.
Parameters
----------
timeseries: obspy.core.stream
The timeseries stream as returned by the call to getWaveform
starttime: obspy.core.UTCDateTime
the starttime of the requested data
endtime: obspy.core.UTCDateTime
the endtime of the requested data
Notes: the original timeseries object is changed.
"""
for trace in timeseries:
trace_starttime = UTCDateTime(trace.stats.starttime)
trace_endtime = UTCDateTime(trace.stats.endtime)
if trace.stats.starttime > starttime:
cnt = int((trace_starttime - starttime) / trace.stats.delta)
trace.data = numpy.concatenate([
numpy.full(cnt, numpy.nan, dtype=numpy.float64),
trace.data])
trace.stats.starttime = starttime
if trace_endtime < endtime:
cnt = int((endtime - trace_endtime) / trace.stats.delta)
trace.data = numpy.concatenate([
trace.data,
numpy.full(cnt, numpy.nan, dtype=numpy.float64)])
trace.stats.endttime = endtime
def crossval_predict(predictor, X, y, prefix, n_cv=5):
if not np.array_equal(predictor.classes_, [0, 1]):
raise Exception("classes labels NOT match")
can_pred_proba = common.can_predict_probability(predictor)
n_samples = X.shape[0]
print "totally {} samples, divided into {} folds".format(n_samples, n_cv)
if can_pred_proba:
datas = np.full((n_samples, 2), np.NaN)
headers = ["{}_{}".format(prefix, t) for t in ["proba", "log_proba"]]
yvalidates = pd.DataFrame(datas, columns=headers, index=y.index)
else:
datas = np.full((n_samples, 1), np.NaN)
header = "{}_label".format(prefix)
yvalidates = pd.DataFrame(datas, columns=[header], index=y.index)
folds = StratifiedKFold(y, n_folds=n_cv, shuffle=True, random_state=seed)
for index, (train_index, test_index) in enumerate(folds):
Xtrain, Xtest = X[train_index], X[test_index]
ytrain, ytest = y[train_index], y[test_index]
predictor.fit(Xtrain, ytrain)
if can_pred_proba:
ytest_probas = predictor.predict_proba(Xtest)
pos_proba = ytest_probas[:, 1] # probability for label=1 (Positive)
yvalidates.iloc[test_index, 0] = pos_proba
yvalidates.iloc[test_index, 1] = np.log(pos_proba)
else:
yvalidates.iloc[test_index, 0] = predictor.predict(Xtest)
print "====== cross-validated on {}-fold ======".format(index + 1)
return yvalidates
def calculate_landslide_probability(self, **kwds):
"""Main method of Landslide Probability class.
Method creates arrays for output variables then loops through all
the core nodes to run the method 'calculate_factor_of_safety.'
Output parameters probability of failure, mean relative wetness,
and probability of saturation are assigned as fields to nodes.
"""
# Create arrays for data with -9999 as default to store output
self.mean_Relative_Wetness = np.full(self.grid.number_of_nodes, -9999.)
self.prob_fail = np.full(self.grid.number_of_nodes, -9999.)
self.prob_sat = np.full(self.grid.number_of_nodes, -9999.)
# Run factor of safety Monte Carlo for all core nodes in domain
# i refers to each core node id
for i in self.grid.core_nodes:
self.calculate_factor_of_safety(i)
# Populate storage arrays with calculated values
self.mean_Relative_Wetness[i] = self._soil__mean_relative_wetness
self.prob_fail[i] = self._landslide__probability_of_failure
self.prob_sat[i] = self._soil__probability_of_saturation
# Values can't be negative
self.mean_Relative_Wetness[self.mean_Relative_Wetness < 0.] = 0.
self.prob_fail[self.prob_fail < 0.] = 0.
# assign output fields to nodes
self.grid.at_node["soil__mean_relative_wetness"] = self.mean_Relative_Wetness
self.grid.at_node["landslide__probability_of_failure"] = self.prob_fail
self.grid.at_node["soil__probability_of_saturation"] = self.prob_sat
def testMakeTableExceptions(self):
# Verify that contents is being type-checked and shape-checked.
with self.assertRaises(ValueError):
text_plugin.make_table([])
with self.assertRaises(ValueError):
text_plugin.make_table('foo')
with self.assertRaises(ValueError):
invalid_shape = np.full((3, 3, 3), 'nope', dtype=np.dtype('S3'))
text_plugin.make_table(invalid_shape)
# Test headers exceptions in 2d array case.
test_array = np.full((3, 3), 'foo', dtype=np.dtype('S3'))
with self.assertRaises(ValueError):
# Headers is wrong type.
text_plugin.make_table(test_array, headers='foo')
with self.assertRaises(ValueError):
# Too many headers.
text_plugin.make_table(test_array, headers=['foo', 'bar', 'zod', 'zoink'])
with self.assertRaises(ValueError):
# headers is 2d
text_plugin.make_table(test_array, headers=test_array)
# Also make sure the column counting logic works in the 1d array case.
test_array = np.array(['foo', 'bar', 'zod'])
with self.assertRaises(ValueError):
# Too many headers.
text_plugin.make_table(test_array, headers=test_array)
def a_variations():
yield np.arange(10)
yield np.array([-1.1, np.nan, 2.2])
yield np.array([-np.inf, 5])
yield (4, 2, 5)
yield (1,)
yield np.full(5, 5)
yield [2.2, -2.3, 0.1]
a = np.linspace(-10, 10, 16).reshape(4, 2, 2)
yield a
yield np.asfortranarray(a)
yield a[::-1]
np.random.RandomState(0).shuffle(a)
yield a
yield 6
yield 6.5
yield -np.inf
yield 1 + 4j
yield [2.2, np.nan]
yield [2.2, np.inf]
yield ((4.1, 2.0, -7.6), (4.3, 2.7, 5.2))
yield np.full(5, np.nan)
yield 1 + np.nan * 1j
yield np.nan + np.nan * 1j
yield np.nan
def test_window_safe(self, factor_len):
# all true data set of (days, securities)
data = full(self.default_shape, True, dtype=bool)
class InputFilter(Filter):
inputs = ()
window_length = 0
class TestFactor(CustomFactor):
dtype = float64_dtype
inputs = (InputFilter(), )
window_length = factor_len
def compute(self, today, assets, out, filter_):
# sum for each column
out[:] = np_sum(filter_, axis=0)
results = self.run_graph(
TermGraph({'windowsafe': TestFactor()}),
initial_workspace={InputFilter(): data},
)
# number of days in default_shape
n = self.default_shape[0]
# shape of output array
output_shape = ((n - factor_len + 1), self.default_shape[1])
check_arrays(
results['windowsafe'],
full(output_shape, factor_len, dtype=float64)
)
def show_output(net,testing=False):
global error
global error_divisor
if testing:
time.sleep(TRAINING_DURATION/2)
output = net.get_output()
output_init = output # Only different line
output = [round(x*255) for x in output]
#print "Red: " + str(output[2]) + "\t" + "Green: " + str(output[1]) + "\t" + "Blue: " + str(output[0]) + "\r",
#sys.stdout.flush()
output = np.full((32, 32, 3), output, dtype='uint8')
cv2.imshow("Output", output)
cv2.waitKey(int(1000 * TRAINING_DURATION / 2))
error += abs(testing[2] - output_init[2])
error += abs(testing[0] - output_init[0])
error_divisor += 2
else:
time.sleep(TRAINING_DURATION/2)
output = net.get_output()
output = [round(x*255) for x in output]
#print "Red: " + str(output[2]) + "\t" + "Green: " + str(output[1]) + "\t" + "Blue: " + str(output[0]) + "\r",
#sys.stdout.flush()
output = np.full((32, 32, 3), output, dtype='uint8')
cv2.imshow("Output", output)
cv2.waitKey(int(1000 * TRAINING_DURATION / 2))
def run(oiter):
# ----- Variable for this run -----
log_alpha_0 = all_log_alpha_0[oiter]
print "Running job {0} on {1}".format(oiter + 1, socket.gethostname())
train_images, train_labels, _, _, _ = load_data()
train_images = train_images[:N_data, :]
train_labels = train_labels[:N_data, :]
batch_idxs = BatchList(N_data, batch_size)
iter_per_epoch = len(batch_idxs)
N_weights, _, loss_fun, frac_err = make_nn_funs(layer_sizes, L2_reg)
def indexed_loss_fun(w, idxs):
return loss_fun(w, X=train_images[idxs], T=train_labels[idxs])
V0 = npr.randn(N_weights) * velocity_scale
losses = []
d_losses = []
alpha_0 = np.exp(log_alpha_0)
for N_iters in all_N_iters:
alphas = np.full(N_iters, alpha_0)
betas = np.full(N_iters, beta_0)
npr.seed(1)
W0 = npr.randn(N_weights) * np.exp(log_param_scale)
results = sgd(indexed_loss_fun, batch_idxs, N_iters, W0, V0, alphas, betas)
losses.append(results['loss_final'])
d_losses.append(d_log_loss(alpha_0, results['d_alphas']))
return losses, d_losses
def test_uninterpolated_nan_regions(boundary, normalize_kernel):
#8086
# Test NaN interpolation of contiguous NaN regions with kernels of size
# identical and greater than that of the region of NaN values.
# Test case: kernel.shape == NaN_region.shape
kernel = Gaussian2DKernel(1, 5, 5)
nan_centroid = np.full(kernel.shape, np.nan)
image = np.pad(nan_centroid, pad_width=kernel.shape[0]*2, mode='constant',
constant_values=1)
with pytest.warns(AstropyUserWarning,
match="nan_treatment='interpolate', however, NaN values detected "
"post convolution. A contiguous region of NaN values, larger "
"than the kernel size, are present in the input array. "
"Increase the kernel size to avoid this."):
result = convolve(image, kernel, boundary=boundary, nan_treatment='interpolate',
normalize_kernel=normalize_kernel)
assert(np.any(np.isnan(result)))
# Test case: kernel.shape > NaN_region.shape
nan_centroid = np.full((kernel.shape[0]-1, kernel.shape[1]-1), np.nan) # 1 smaller than kerenel
image = np.pad(nan_centroid, pad_width=kernel.shape[0]*2, mode='constant',
constant_values=1)
result = convolve(image, kernel, boundary=boundary, nan_treatment='interpolate',
normalize_kernel=normalize_kernel)
assert(~np.any(np.isnan(result))) # Note: negation
def complete_obs_table(obs_table, used_columns, filter_list, tolerance,
lim_flag, default_error=0.1, systematic_deviation=0.1):
"""Complete the observation table
For each filter:
* If the corresponding error is not present in the used column list or in
the table columns, add (or replace) an error column with the default
error.
* Adjust the error value.
Parameters
----------
obs_table: astropy.table.Table
The observation table.
used_columns: list of strings
The list of columns to use in the observation table.
filter_list: list of strings
The list of filters used in the analysis.
tolerance: float
Tolerance threshold under flux error is considered as 0.
lim_flag: boolean
Do we process upper limits (True) or treat them as no-data (False)?
default_error: float
Default error factor used when the provided error in under the
tolerance threshold.
systematic_deviation: float
Systematic deviation added to the error.
Returns
-------
obs_table = astropy.table.Table
The completed observation table
Raises
------
Exception: When a filter is not present in the observation table.
"""
# TODO Print or log a warning when an error column is in the used column
# list but is not present in the observation table.
for name in filter_list:
if name not in obs_table.columns:
raise Exception("The filter <{}> (at least) is not present in "
"the observation table.".format(name))
name_err = name + "_err"
if name_err not in obs_table.columns:
obs_table.add_column(Column(name=name_err,
data=np.full(len(obs_table), np.nan)),
index=obs_table.colnames.index(name)+1)
elif name_err not in used_columns:
obs_table[name_err] = np.full(len(obs_table), np.nan)
obs_table[name], obs_table[name_err] = adjust_data(obs_table[name],
obs_table[name_err],
tolerance,
lim_flag,
default_error,
systematic_deviation)
return obs_table
def loop_over_zone(self, array):
"""Generate a masked array and retrieve average values.
:param array: ndarray, representing zone.
:returns: ndarray, average values for zone.
"""
mesh_1 = self.model_data.model_mesh3D[1]
rows = mesh_1.shape[0]
arr_zoned = [np.full(mesh_1.shape[1:3], np.nan)] * int(np.max(mesh_1))
for zone in range(int(np.max(mesh_1))):
temp = np.array([np.full(mesh_1.shape[1:3], np.nan)] * rows)
zone_1 = float(zone + 1)
for layer in range(rows):
mesh_1_layer = mesh_1[layer]
temp[layer][mesh_1_layer == zone_1] = array[layer][mesh_1_layer == zone_1]
# End for
masked_temp = np.ma.masked_array(temp, np.isnan(temp))
arr_zoned[zone] = np.mean(masked_temp, axis=0)
# End for
return arr_zoned
def _init(self, X, lengths, params):
init = 1. / self.n_components
if 's' in params or not hasattr(self, "startprob_"):
self.startprob_ = np.full(self.n_components, init)
if 't' in params or not hasattr(self, "transmat_"):
self.transmat_ = np.full((self.n_components, self.n_components),
init)
def iterations(X, y, T): d,n=X.shape e0=1.0 f0=1.0 a0=np.full(d, 10e-16) b0=np.full(d, 10e-16) mu0=np.ones((d,1)) sigma0=np.ones((d,d)) L=np.zeros(T) Eln_qw=np.zeros(T) Eln_qlambda=np.zeros(T) Eln_qalpha=np.zeros(T) Eln_pw=np.zeros(T) Eln_plambda=np.zeros(T) Eln_palpha=np.zeros(T) Eln_py=np.zeros(T) for t in range(T): a,b,e,f,mu,sigma=updates(X,y,a0,b0,e0,f0,mu0,sigma0) L[t]=obj_function(X,y,a,b,e,f,mu,sigma) if t<T: a0=a b0=b e0=e f0=f mu0=mu sigma0=sigma return a, b, e, f, L, mu
def initBuffers(self,puzzle):
#define lengths buffer and copy to the GPU
#as we will not read from this buffer later, mapping is not required
self.lengths = np.full(self.simulations,np.iinfo(np.int16).max,dtype=np.int16)
self.lengthsBuffer = cl.Buffer(self.context, cl.mem_flags.READ_WRITE | cl.mem_flags.COPY_HOST_PTR, hostbuf=self.lengths)
#define buffer for aggregated lengths for each workgroup
self.groupLengths = np.full(self.workGroups,np.iinfo(np.int16).max,dtype=np.int16)
self.groupLengthsBuffer = cl.Buffer(self.context, cl.mem_flags.READ_WRITE | cl.mem_flags.USE_HOST_PTR, hostbuf=self.groupLengths)
#map group lengths buffer
cl.enqueue_map_buffer(self.queue,self.groupLengthsBuffer,cl.map_flags.READ,0,self.groupLengths.shape,self.groupLengths.dtype)
#get the input puzzle ready for the kernel; convert to 8 bit int (char)
p = np.array(puzzle['puzzle']).astype(np.int8)
#subtract 1 so that -1 denotes a gap and 0 denotes a square to be filled
p = p - np.ones_like(p,dtype=p.dtype)
#copy the puzzle, one for each simulation
self.puzzles = np.zeros((self.simulations,self.height,self.width),dtype=p.dtype)
self.puzzles[:,0:self.height,0:self.width] = p
#define puzzles buffer and copy data (we do not need to worry about getting data out of this buffer, so mapping isn't required)
#this buffer contains the input puzzles, one for each invocation (the puzzle is too large to hold in local or shared memory)
self.puzzlesFlattened = self.puzzles.ravel()
self.puzzlesBuffer = cl.Buffer(self.context, cl.mem_flags.READ_WRITE | cl.mem_flags.COPY_HOST_PTR, hostbuf=self.puzzlesFlattened)
#define output buffer for best solutions aggregated across workgroups
self.solutions = self.puzzles[0:self.workGroups]
self.solutionsFlattened = self.solutions.ravel()
self.solutionsBuffer = cl.Buffer(self.context, cl.mem_flags.READ_WRITE | cl.mem_flags.USE_HOST_PTR, hostbuf=self.solutionsFlattened)
#map solutions buffer
cl.enqueue_map_buffer(self.queue,self.solutionsBuffer,cl.map_flags.READ,0,self.solutionsFlattened.shape,self.solutions.dtype)
def filter2d(input_img, filter, size = 3):
newimg = Image.new(input_img.mode, input_img.size)
data = list(input_img.getdata())
pixels = np.reshape(data, (input_img.size[1], input_img.size[0]))
a = size/2
b = size/2
imgheight, imgwidth = input_img.size
if filter == 'average':
weight = np.full(size*size, float(1)/(size*size))
if filter == 'laplacian':
laplacian = np.array([[-1, -1, -1],
[-1, 8, -1],
[-1, -1, -1]])
weight = laplacian.flatten()
if filter == 'sobel1':
sobel1 = np.array([[-1, -2, -1],
[0, 0, 0],
[1, 2, 1]])
weight = sobel1.flatten()
if filter == 'sobel2':
sobel2 = np.array([[-1, 0, 1],
[-2, 0, 2],
[-1, 0, 1]])
weight = sobel2.flatten()
for y in range(input_img.size[1]):
for x in range(input_img.size[0]):
z = np.full(size * size, pixels[y][x])
for j in range(y - a, y + a + 1):
for i in range(x - b, x + b + 1):
if i > 0 and i < imgheight and j > 0 and j < imgwidth:
z[(j - y + a) * size + i - x + b] = pixels[j][i]
newimg.putpixel((x, y), np.dot(weight, z))
return newimg
def test_multiple_rolling_factors(self):
loader = self.loader
engine = SimpleFFCEngine(loader, self.dates, self.asset_finder)
shape = num_dates, num_assets = (5, len(self.assets))
dates = self.dates[10:10 + num_dates]
short_factor = RollingSumDifference(window_length=3)
long_factor = RollingSumDifference(window_length=5)
high_factor = RollingSumDifference(
window_length=3,
inputs=[USEquityPricing.open, USEquityPricing.high],
)
results = engine.factor_matrix(
{'short': short_factor, 'long': long_factor, 'high': high_factor},
dates[0],
dates[-1],
)
self.assertEqual(set(results.columns), {'short', 'high', 'long'})
# row-wise sum over an array whose values are all (1 - 2)
assert_array_equal(
results['short'].unstack().values,
full(shape, -short_factor.window_length),
)
assert_array_equal(
results['long'].unstack().values,
full(shape, -long_factor.window_length),
)
# row-wise sum over an array whose values are all (1 - 3)
assert_array_equal(
results['high'].unstack().values,
full(shape, -2 * high_factor.window_length),
)
def generateFileList(rootPath):
iterNum = 0
X_train_i = np.array([])
X_test_i = np.array([])
y_train_i = np.array([])
y_test_i = np.array([])
if reducedInput:
print "WARN: not using all available input"
X_train_path = ["004_M3", "012_F2"]
y_train_data = [1, 0]
else:
global X_train_path, y_train_data
for p in X_train_path:
a = root + "/" + p + "/C" + p + "_INDE"
b = root + "/" + p + "/C" + p + "_SENT"
filesThisFolder = np.append([a + "/" + f for f in listdir(a) if isfile(join(a, f)) and f.endswith(".wav")],
[b + "/" + f for f in listdir(b) if isfile(join(b, f)) and f.endswith(".wav")])
y_train_i = np.append(y_train_i, np.full((filesThisFolder.shape[0]), y_train_data[iterNum], dtype='int8'))
X_train_i = np.append(X_train_i, filesThisFolder)
iterNum += 1
iterNum = 0
for p in X_test_path:
a = root + "/" + p + "/C" + p + "_INDE"
b = root + "/" + p + "/C" + p + "_SENT"
filesThisFolder = np.append([a + "/" + f for f in listdir(a) if isfile(join(a, f)) and f.endswith(".wav")],
[b + "/" + f for f in listdir(b) if isfile(join(b, f)) and f.endswith(".wav")])
y_test_i = np.append(y_test_i, np.full((filesThisFolder.shape[0]), y_test_data[iterNum], dtype='int8'))
X_test_i = np.append(X_test_i, filesThisFolder)
iterNum += 1
return ((X_train_i, y_train_i), (X_test_i, y_test_i))
def run():
train_images, train_labels, _, _, _ = load_data()
train_images = train_images[:N_data, :]
train_labels = train_labels[:N_data, :]
batch_idxs = BatchList(N_data, batch_size)
iter_per_epoch = len(batch_idxs)
N_weights, _, loss_fun, frac_err = make_nn_funs(layer_sizes, L2_reg)
def indexed_loss_fun(w, idxs):
return loss_fun(w, X=train_images[idxs], T=train_labels[idxs])
log_alphas = np.full(N_iters, log_alpha_0)
betas = np.full(N_iters, beta_0)
npr.seed(1)
V0 = npr.randn(N_weights) * velocity_scale
W0 = npr.randn(N_weights) * np.exp(log_param_scale)
output = []
for i in range(N_meta_iter):
print "Meta iteration {0}".format(i)
results = sgd(indexed_loss_fun, batch_idxs, N_iters,
W0, V0, np.exp(log_alphas), betas, record_learning_curve=True)
learning_curve = results['learning_curve']
d_log_alphas = np.exp(log_alphas) * results['d_alphas']
output.append((learning_curve, log_alphas, d_log_alphas))
log_alphas = log_alphas - meta_alpha * d_log_alphas
return output
def test_many_inputs(self):
"""
Test adding NumericalExpressions with >10 inputs.
"""
# Create an initial NumericalExpression by adding two factors together.
f = self.f
expr = f + f
self.fake_raw_data = {f: full((5, 5), 0, float)}
expected = 0
# Alternate between adding and subtracting factors. Because subtraction
# is not commutative, this ensures that we are combining factors in the
# correct order.
ops = (add, sub)
for i, name in enumerate(ascii_uppercase):
op = ops[i % 2]
NewFactor = type(
name,
(Factor,),
dict(dtype=float64_dtype, inputs=(), window_length=0),
)
new_factor = NewFactor()
# Again we need a NumericalExpression, so add two factors together.
new_expr = new_factor + new_factor
self.fake_raw_data[new_factor] = full((5, 5), i + 1, float)
expr = op(expr, new_expr)
# Double the expected output since each factor is counted twice.
expected = op(expected, (i + 1) * 2)
self.check_output(expr, full((5, 5), expected, float))
def make_default_configuration(self):
self.global_register = ccpdv4['CCPD_GLOBAL'].copy()
self.pixel_register = {
"threshold": np.full((48, 12), 7, dtype=np.uint8), # 16 columns (triple col) x 6 rows (double row)
# "monitor": value = np.full((48,12), 0, dtype=np.uint8),
"injection": np.full((6, ), 0, dtype=np.uint8)
}
def test_rolling_and_nonrolling(self):
open_ = USEquityPricing.open
close = USEquityPricing.close
volume = USEquityPricing.volume
# Test for thirty days up to the last day that we think all
# the assets existed.
dates_to_test = self.dates[-30:]
constants = {open_: 1, close: 2, volume: 3}
loader = PrecomputedLoader(constants=constants, dates=self.dates, sids=self.asset_ids)
engine = SimplePipelineEngine(lambda column: loader, self.dates, self.asset_finder)
sumdiff = RollingSumDifference()
result = engine.run_pipeline(
Pipeline(
columns={"sumdiff": sumdiff, "open": open_.latest, "close": close.latest, "volume": volume.latest}
),
dates_to_test[0],
dates_to_test[-1],
)
self.assertIsNotNone(result)
self.assertEqual({"sumdiff", "open", "close", "volume"}, set(result.columns))
result_index = self.asset_ids * len(dates_to_test)
result_shape = (len(result_index),)
check_arrays(result["sumdiff"], Series(index=result_index, data=full(result_shape, -3, dtype=float)))
for name, const in [("open", 1), ("close", 2), ("volume", 3)]:
check_arrays(result[name], Series(index=result_index, data=full(result_shape, const, dtype=float)))
def _recognize3(self, scores, transitions):
lengthT = scores.shape[0]
lengthS = transitions.shape[1]
cost = np.full((lengthT, lengthT), np.inf, 'float32')
back = np.full((lengthT, lengthT), np.inf, 'int32')
cost[0] = np.min(scores[0])
back[0] = -1
transcript = []
attention = []
for s in xrange(1, lengthT):
for t in xrange(min(s * lengthS, lengthT)):
#if s % self.nstates == 0: # end state
cost[s, t] = np.min(scores[s])
q = transitions[t].copy()
q[:min(t,lengthS)] += cost[s - 1, t - min(t,lengthS) : t]
back[s, t] = q.argmin() + 1
cost[s, t] += q.min()
t = lengthT - 1
s = 1
while t >= 0 and s < lengthT:
if s % self.nstates == 0:
attention.append(t)
transcript.append(scores[t].argmin() / self.nstates)
t -= back[-s, t]
s += 1
return transcript[::-1], attention[::-1]
def run():
train_images, train_labels, _, _, _ = load_data(normalize=True)
train_images = train_images[:N_real_data, :]
train_labels = train_labels[:N_real_data, :]
batch_idxs = BatchList(N_fake_data, batch_size)
parser, _, loss_fun, frac_err = make_nn_funs(layer_sizes, L2_reg, return_parser=True)
N_weights = parser.N
fake_data = npr.randn(*(train_images[:N_fake_data, :].shape)) * init_fake_data_scale
fake_labels = one_hot(np.array(range(N_fake_data)) % N_classes, N_classes) # One of each.
def indexed_loss_fun(x, meta_params, idxs): # To be optimized by SGD.
return loss_fun(x, X=meta_params[idxs], T=fake_labels[idxs])
def meta_loss_fun(x): # To be optimized in the outer loop.
return loss_fun(x, X=train_images, T=train_labels)
log_alphas = np.full(N_iters, log_alpha_0)
betas = np.full(N_iters, beta_0)
npr.seed(0)
v0 = npr.randn(N_weights) * velocity_scale
x0 = npr.randn(N_weights) * np.exp(log_param_scale)
output = []
for i in range(N_meta_iter):
results = sgd2(indexed_loss_fun, meta_loss_fun, batch_idxs, N_iters,
x0, v0, np.exp(log_alphas), betas, fake_data)
learning_curve = results['learning_curve']
validation_loss = results['M_final']
output.append((learning_curve, validation_loss, fake_data))
fake_data -= results['dMd_meta'] * data_stepsize # Update data with one gradient step.
print "Meta iteration {0} Valiation loss {1}".format(i, validation_loss)
return output
def load_model(nbins_sfh=7,sigma=0.3,df=2.,agelims=None,objname=None, **extras):
# we'll need this to access specific model parameters
n = [p['name'] for p in model_params]
# replace nbins_sfh
nbins_sfh = 4 + (int(objname)-1) / 9
# create SFH bins
zred = model_params[n.index('zred')]['init']
tuniv = WMAP9.age(zred).value
# now construct the nonparametric SFH
# current scheme: six bins, four spaced equally in logarithmic
# last bin is 15% age of the Universe, first two are 0-30, 30-100
tbinmax = (tuniv*0.85)*1e9
agelims = agelims[:2] + np.linspace(agelims[2],np.log10(tbinmax),nbins_sfh-2).tolist() + [np.log10(tuniv*1e9)]
agebins = np.array([agelims[:-1], agelims[1:]])
# load nvariables and agebins
model_params[n.index('agebins')]['N'] = nbins_sfh
model_params[n.index('agebins')]['init'] = agebins.T
model_params[n.index('mass')]['N'] = nbins_sfh
model_params[n.index('logsfr_ratios')]['N'] = nbins_sfh-1
model_params[n.index('logsfr_ratios')]['init'] = np.full(nbins_sfh-1,0.0) # constant SFH
model_params[n.index('logsfr_ratios')]['prior'] = priors.StudentT(mean=np.full(nbins_sfh-1,0.0),
scale=np.full(nbins_sfh-1,sigma),
df=np.full(nbins_sfh-1,df))
return sedmodel.SedModel(model_params)
def plot_risk(risk_local, risk_central, risk_dist, iters, name_file, label_x, label_y):
size = iters.shape[0]
# Create the data
risk_local = np.full(size, risk_local)
risk_central = np.full(size, risk_central)
iters = np.array(iters)
# Plot graphs
sns.set_style("ticks")
plt.figure()
plt.rc('text', usetex = True)
plt.rc('text.latex', unicode = True)
tests = list(risk_dist.keys())
with sns.color_palette("tab10", len(tests) + 2):
for test in tests:
label = "SVM distribuído com " + test
plt.plot(iters, risk_dist[test], linewidth = 2, label = label)
plt.plot(iters, risk_local, linewidth = 2.2, linestyle = '-.', label = 'SVM Local')
plt.plot(iters, risk_central, linewidth = 2.2, linestyle = '-.', label = 'SVM Central')
plt.legend(loc = 'upper right')
sns.despine()
plt.xlabel(label_x)
plt.ylabel(label_y)
file = str(plots_path) + "/" + name_file + ".pdf"
plt.savefig(file, transparent = True)
def gen(npslice, output_type, black_filename, imagenet_filename):
sets = ['suzanne','tux', 'airplane']
imagenet = []
black = []
for i, s in enumerate(sets):
print("set", s)
input_samples = db.load('../00_Blender/%s_s1.h5' % s)
assert np.max(input_samples['images']) > 50
size = input_samples['alphas'][npslice].shape[0]
min, max = range_extend_min_max(input_samples['images'], input_samples['alphas'])
print("range_extend: min", min, "max", max)
range_extended_images = range_extend(input_samples['images'][npslice],
input_samples['alphas'][npslice], min, max)
range_extended_db = {
'images': range_extended_images,
'alphas': input_samples['alphas'][npslice],
'params': input_samples['params'][npslice],
'label': np.full((size), i, dtype='uint8'),
'label_split': np.full((size, len(sets)), [x == i for x in range(len(sets))], dtype='uint8')
}
bg = bgp.ImageNet(output_type)
newdb = generate_dataset(range_extended_db, bg.generator(), size)
newdb['background_indexes'] = np.array(bg.provided_indexes)
imagenet.append(newdb)
black.append(range_extended_db)
db.write(imagenet_filename, append(imagenet))
db.write(black_filename, append(black))
def _expected_kid_and_std(real_imgs, gen_imgs, max_block_size=1024):
n_r, dim = real_imgs.shape
n_g = gen_imgs.shape[0]
n_blocks = int(np.ceil(max(n_r, n_g) / max_block_size))
sizes_r = np.full(n_blocks, n_r // n_blocks)
to_patch = n_r - n_blocks * (n_r // n_blocks)
if to_patch > 0:
sizes_r[-to_patch:] += 1
inds_r = np.r_[0, np.cumsum(sizes_r)]
assert inds_r[-1] == n_r
sizes_g = np.full(n_blocks, n_g // n_blocks)
to_patch = n_g - n_blocks * (n_g // n_blocks)
if to_patch > 0:
sizes_g[-to_patch:] += 1
inds_g = np.r_[0, np.cumsum(sizes_g)]
assert inds_g[-1] == n_g
ests = []
for i in range(n_blocks):
r = real_imgs[inds_r[i]:inds_r[i + 1]]
g = gen_imgs[inds_g[i]:inds_g[i + 1]]
k_rr = (np.dot(r, r.T) / dim + 1)**3
k_rg = (np.dot(r, g.T) / dim + 1)**3
k_gg = (np.dot(g, g.T) / dim + 1)**3
ests.append(-2 * k_rg.mean() +
k_rr[np.triu_indices_from(k_rr, k=1)].mean() +
k_gg[np.triu_indices_from(k_gg, k=1)].mean())
var = np.var(ests, ddof=1) if len(ests) > 1 else np.nan
return np.mean(ests), np.sqrt(var / len(ests))
@author: Amarantine
"""
import numpy as np
import os
import pickle
MeditationSessions=['par011_1','par020_2','par040_2','par050_2','par061_1',
'par071_1','par080_2','par090_2','par101_1','par111_1',
'par120_2','par130_2']
NoMeditationSessions=['par011_2','par020_1','par040_1','par050_1','par061_2',
'par071_2','par080_1','par090_1','par101_2','par111_2',
'par120_1','par130_1']
meta_roc_auc_mean=np.full((2,12,12),np.nan) # 2 sessions, 12 participatnts, 12 trials
for participant, location in enumerate(MeditationSessions):
participant=participant
path='E:\Biomedical.master\Data\\' + location
os.chdir(path)
print path
with open ('analysisResults_NumOftrials', 'rb') as fp:
analysisResults_NumOftrials = pickle.load(fp)
meta_roc_auc_mean[1][participant][:]=analysisResults_NumOftrials['meta_roc_auc_mean']
for participant, location in enumerate(NoMeditationSessions):
participant=participant
path='E:\Biomedical.master\Data\\' + location
os.chdir(path)
print path
with open ('analysisResults_NumOftrials', 'rb') as fp:
data3 = {
'column1' : [110, 111, 113, 115],
'column2' : [200, 300, 400, 500],
'column3' : [10, 20, 30, 40],
}
df3 = pd.DataFrame(data3, index=['p1', 'p2', 'p3', 'p4'])
print('การกำหนดข้อมูลแบบดิกชันนารี')
print(df3)
data4 = {
'Mon' : np.arange(10, 14),
'Tue' : np.random.randint(10, 100, 4),
'Web' : np.full(4, 10)
}
df4 = pd.DataFrame(data4, index=['p1', 'p2', 'p3', 'p4'])
print('การกำหนดข้อมูลแบบดิกชันนารี โดยใช้ numpy')
print(df4)
### การ join dataframe คือการนำ dataframe / dataframe มาต่อกัน
data5 = {
'col 1' : [10, 20, 30],
'col 2' : [40, 50, 60],
'col 3' : [70, 80, 90]
}
df5 = pd.DataFrame(data5, index=list('ABC'))
data6 = {
def _extract(self):
# Extract all trials...
# Get all stim_sync events detected
ttls = [raw.get_port_events(tr, 'BNC1') for tr in self.bpod_trials]
# Report missing events
n_missing = sum(len(pulses) != 3 for pulses in ttls)
# Check if all stim syncs have failed to be detected
if n_missing == len(ttls):
_logger.error(f'{self.session_path}: Missing ALL BNC1 TTLs ({n_missing} trials)')
elif n_missing > 0: # Check if any stim_sync has failed be detected for every trial
_logger.warning(f'{self.session_path}: Missing BNC1 TTLs on {n_missing} trial(s)')
# Extract datasets common to trainingChoiceWorld
training = [ContrastLR, FeedbackTimes, Intervals, GoCueTimes, StimOnTriggerTimes]
out, _ = run_extractor_classes(training, session_path=self.session_path, save=False,
bpod_trials=self.bpod_trials, settings=self.settings)
# GoCueTriggerTimes is the same event as StimOnTriggerTimes
out['goCueTrigger_times'] = out['stimOnTrigger_times'].copy()
# StimCenterTrigger times
# Get the stim_on_state that triggers the onset of the stim
stim_center_state = np.array([tr['behavior_data']['States timestamps']
['stim_center'][0] for tr in self.bpod_trials])
out['stimCenterTrigger_times'] = stim_center_state[:, 0].T
# StimCenter times
stim_center_times = np.full(out['stimCenterTrigger_times'].shape, np.nan)
for i, (sync, last) in enumerate(zip(ttls, out['stimCenterTrigger_times'])):
"""We expect there to be 3 pulses per trial; if this is the case, stim center will
be the third pulse. If any pulses are missing, we can only be confident of the correct
one if exactly one pulse occurs after the stim center trigger"""
if len(sync) == 3 or (len(sync) > 0 and sum(pulse > last for pulse in sync) == 1):
stim_center_times[i] = sync[-1]
out['stimCenter_times'] = stim_center_times
# StimOn times
stimOn_times = np.full(out['stimOnTrigger_times'].shape, np.nan)
for i, (sync, last) in enumerate(zip(ttls, out['stimCenterTrigger_times'])):
"""We expect there to be 3 pulses per trial; if this is the case, stim on will be the
second pulse. If 1 pulse is missing, we can only be confident of the correct one if
both pulses occur before the stim center trigger"""
if len(sync) == 3 or (len(sync) == 2 and sum(pulse < last for pulse in sync) == 2):
stimOn_times[i] = sync[1]
out['stimOn_times'] = stimOn_times
# RewardVolume
trial_volume = [x['reward_amount'] for x in self.bpod_trials]
out['rewardVolume'] = np.array(trial_volume).astype(np.float64)
# StimOffTrigger times
# StimOff occurs at trial start (ignore the first trial's state update)
out['stimOffTrigger_times'] = np.array(
[tr["behavior_data"]["States timestamps"]
["trial_start"][0][0] for tr in self.bpod_trials[1:]]
)
# StimOff times
"""
There should be exactly three TTLs per trial. stimOff_times should be the first TTL pulse.
If 1 or more pulses are missing, we can not be confident of assigning the correct one.
"""
trigg = out['stimOffTrigger_times']
out['stimOff_times'] = np.array([sync[0] if len(sync) == 3 else np.nan
for sync, off in zip(ttls[1:], trigg)])
# FeedbackType is always positive
out['feedbackType'] = np.ones(len(out['feedback_times']), dtype=np.int8)
# ItiIn times
out['itiIn_times'] = np.array(
[tr["behavior_data"]["States timestamps"]
["iti"][0][0] for tr in self.bpod_trials]
)
# NB: We lose the last trial because the stim off event occurs at trial_num + 1
n_trials = out['stimOff_times'].size
return [out[k][:n_trials] for k in self.var_names]
#data=smallset
#n_user = 100
#n_artist = 100
data=realset
n_user = 1882
n_artist = 3000
sparity_ratio = data.size / ((n_user*n_artist) - data.size)
f=3
l=0.01
pref = np.full(data.size, 1.)
user_mat = sp.csr_matrix(np.full((n_user, f), 0.5))
artist_mat = sp.csr_matrix((n_artist, f))
lambda_identity = l*sp.identity(f)
p_artist = sp.csr_matrix((pref, (data['artistID'], data['userID'])), shape=(n_artist, n_user)).toarray()
p_user = p_artist.transpose()
sparity_vec = 1 + data['interactions']*sparity_ratio
artist_conf = []
artist_conf_sparse = []
idx = 0
while idx < n_artist:
c_vec = np.full(n_user, 0.)
filter = data['artistID']==idx
for i in data[filter]:
def Question_2_essential(STDP_Mode,Inuput_Rate):
# Question_1()
# Units
ms = 0.001
mv = 0.001
MO = 1.0e6
nA = 1.0e-9
nS = 1.0e-9
# Neuron Param
tau_m = 10*ms
EL = -65*mv
V_reset = -65*mv
Vth = -50*mv
Rm = 100*MO
Ie = 0
N = 40
Recent_Post_Spike = -1000
# Synapses
tau_s = 2*ms
gi = 4*nS
ga = 2.08*nS
Es = 0
Deta_s = 0.5
# 40 incoming synapses
S = np.zeros(N)
# g = np.full(N,gi)
# g = np.full((1200005,N),0)
# g[0,:] = gi
Spikes_Counts = np.zeros(N)
Recent_Pre_Spikes = np.zeros(N)
Deta_t = 0.25*ms
r_ini = Inuput_Rate #15 Hz
# SDTP par
A_plus = 0.2*nS
A_minus = 0.25*nS
tau_plus = 20*ms
tau_minus = 20*ms
# Presynaptic Spike time:
# t_pre = 0
# Postsynaptic Spike time:
# t_post = 0
# Set the SDTP flag:
# STDP_Mode = True
# STDP_Mode = Flase
if STDP_Mode:
g = np.full(N,gi)
else:
g = np.full(N,ga)
# print("The g_i is : ",g)
# print("The S is : ",S)
# Calculate the Time interval
Runtime = 300
Spike_Average_bin = 10
TimeSteps = math.ceil(Runtime/Deta_t)
print("TimeSteps: ", TimeSteps)
V = []
V_old = V_reset
print("The g_i is : ",g)
# Pre-synaptic neurons N:
Spike_Count = 0
Spike_Average = []
g_average = []
for t in range(0,TimeSteps):
I_s = 0
# Apply a possion process use random function
# I_s = Rm*(Es - V_old)*np.sum(S*g).all()
for it in range(0,40):
I_s += Rm*(Es - V_old)*(S[it]*g[it])
for i in range(0,40):
rand = random.uniform(0,1)
# Spike occures
if rand < Deta_t*r_ini:
S[i] = S[i] + 0.5
# print("Pre_Spike")
# Spikes_Counts[i] += 1
if STDP_Mode:
# Store the Pre Spike time
Recent_Pre_Spikes[i] = t
STDP_Deta_t = (Recent_Post_Spike - Recent_Pre_Spikes[i])
# if STDP_Deta_t <= 0:
g[i] = g[i] - A_minus * math.exp(-abs((STDP_Deta_t)*Deta_t)/tau_minus)
if g[i] < 0:
g[i] = 0
# Spike not occures
else:
S[i] = S[i]-S[i]*Deta_t/tau_s
# Update The neuron:
# Post-Synaptic neurons
# I_s = Rm*(Es - V_old)*np.sum(S*g)
V_new = V_old + Deta_t*((EL - V_old + Rm*Ie+I_s) / tau_m)
# If there is a spike and update the V value
if V_new > Vth:
V_new = V_reset
Recent_Post_Spike = t
Spike_Count +=1
# print("Post Spike!!")
if STDP_Mode:
for p in range(0,40):
STDP_Deta_t = (Recent_Post_Spike - Recent_Pre_Spikes[i])
# if STDP_Deta_t > 0:
g[p] = g[p] + A_plus * math.exp(-abs((STDP_Deta_t)*Deta_t)/tau_plus)
if g[p] > (4*nS):
g[p] = 4*nS
V_old = V_new
V.append(V_new)
if (t+1)%40000 == 0:
Spike_Average.append(Spike_Count/Spike_Average_bin)
# print("Spike_Average",Spike_Average)
Spike_Count = 0
if t > 800000:
g_average.append(np.mean(g))
g_average_all = (np.array(g_average)).mean()
Spike_Average = np.array(Spike_Average)
return g,Spike_Average,g_average_all
def main():
parser = argparse.ArgumentParser(
description='Worker script for the case study.')
# Experiment settings
descriptors = [
'Bakery', 'Sour', 'Intensity', 'Sweet', 'Burnt', 'Pleasantness',
'Fish', 'Fruit', 'Garlic', 'Spices', 'Cold', 'Acid', 'Warm', 'Musky',
'Sweaty', 'Ammonia', 'Decayed', 'Wood', 'Grass', 'Flower', 'Chemical'
]
parser.add_argument('--descriptor',
choices=descriptors,
default='Bakery',
help='The descriptor type to get p-values for.')
parser.add_argument('--fdr_threshold',
type=float,
default=0.2,
help='Target false discovery rate.')
parser.add_argument(
'--importance_threshold',
type=float,
default=1e-3,
help=
'Minimum heuristic feature importance to make a feature test-worthy.')
# Get the arguments from the command line
args = parser.parse_args()
dargs = vars(args)
torch.set_num_threads(1) # bad torch, no biscuit
# Load the data and the model
print('Loading data')
X, Y, descriptors, target_features = load_olfaction()
features = X.columns
print('Loading model')
# Get the model and data specifically for this descriptor class
x, y, forest_model = load_or_fit_model(args.descriptor, X, Y)
# Handle an idiosyncracy of multiprocessing with sklearn random forests
for m in forest_model.models:
m.n_jobs = 1
# Get the weights to use in feature ranking
model_weights = get_model_weights(forest_model)
print('Total: {}'.format(len(X.columns)))
all_p_path = 'data/p_{}.npy'.format(args.descriptor)
if os.path.exists(all_p_path):
p_values = np.load(all_p_path)
else:
p_values = np.full(len(features), np.nan)
for feat_idx in range(len(features)):
if not np.isnan(p_values[feat_idx]):
continue
p_path = 'data/p_{}_{}.npy'.format(args.descriptor, feat_idx)
# Check if we have already computed this p-value
if os.path.exists(p_path):
p_values[feat_idx] = np.load(p_path)
# print('p-value for {}: {}'.format(features[feat_idx], p_values[feat_idx]))
continue
# Check if the model assigns zero weight to this feature such that it can be ignored
if np.abs(model_weights[feat_idx]) < args.importance_threshold:
# print('p-value for {}: 1 (0 weight in model)'.format(features[feat_idx]))
p_values[feat_idx] = 1
continue
print('************ Missing p-value for {} ************'.format(
feat_idx))
# Save the aggregated results
np.save(all_p_path, p_values)
# Print the top-ranked features by their heuristic weight
for rank, (target_feature,
importance) in enumerate(target_features[args.descriptor]):
p_value = p_values[features.get_loc(target_feature)]
print('{} & {:.4f} & {:.4f} \\\\'.format(
target_feature.replace('\'', ''), importance, p_value))
if np.any(np.isnan(p_values)):
print('{} NaN p-values!'.format(np.isnan(p_values).sum()))
missing = np.where(np.isnan(p_values))[0]
print(missing)
print([features[idx] for idx in missing])
print('Setting to 1')
p_values[np.isnan(p_values)] = 1
# Only consider features with substantial heuristic feature importance
important = model_weights >= args.importance_threshold
p_important = p_values[important]
print('{} features above importance threshold'.format(important.sum()))
# Multiple testing correction via Benjamini-Hochberg at a 20% FDR
# discoveries = bh(p_values, args.fdr_threshold) # Test all features
discoveries = np.arange(len(features))[important][bh(
p_important, args.fdr_threshold)] # Test important features
discovery_features = features[discoveries]
discovery_p = p_values[discoveries]
discovery_weights = model_weights[discoveries]
# Print the discoveries along with their model weights and p-values
order = np.argsort(np.abs(discovery_weights))[::-1]
print('')
print('Molecular Feature & Model Weight & $p$-value \\\\')
for f, w, p in zip(discovery_features[order], discovery_weights[order],
discovery_p[order]):
print('{} & {:.4f} & {:.4f} \\\\'.format(f, w, p))
def COMSM2127(STDP_Mode):
# Pre-Setting
B = 0 # 0Hz
# Units
ms = 0.001
mv = 0.001
MO = 1.0e6
nA = 1.0e-9
nS = 1.0e-9
# Neuron Param
tau_m = 10*ms
EL = -65*mv
V_reset = -65*mv
Vth = -50*mv
Rm = 100*MO
Ie = 0
N = 40
Recent_Post_Spike = 0
# Synapses
tau_s = 2*ms
gi = 4*nS
ga = 2.08*nS
Es = 0
Deta_s = 0.5
# 40 incoming synapses
S = np.zeros(N)
Spikes_Counts = np.zeros(N)
Recent_Pre_Spikes = np.zeros(N)
Deta_t = 0.25*ms
# r_ini = Inuput_Rate #15 Hz
# SDTP par
A_plus = 0.2*nS
A_minus = 0.25*nS
tau_plus = 20*ms
tau_minus = 20*ms
if STDP_Mode:
g = np.full(N,gi)
else:
g = np.full(N,gi)
# Calculate the Time interval
Runtime = 300
TimeSteps = math.ceil(Runtime/Deta_t)
print("TimeSteps: ", TimeSteps)
V = []
V_old = V_reset
print("The g_i is : ",g)
# Question4_Par
freq = 10 # 10Hz
r_0 = 15 # 20Hz
# Pre-synaptic neurons N:
Spike_Count = 0
# example_pre_index = 17
Pre_Neuron_Spikes_20s = []
Pre_Neuron_Spikes_200s = []
for example in range(0,N,1):
Pre_Neuron_Spikes_20s.append([])
Pre_Neuron_Spikes_200s.append([])
Post_Neuron_Spikes_20s = []
Post_Neuron_Spikes_200s = []
for t in range(0,TimeSteps):
I_s = 0
# <r>(t) = <r>0 + B*sin(2*pai*frequency*t)
r_ini = r_0 + B * math.sin(2*(math.pi)*freq*((t+1)*Deta_t))
# Apply a possion process use random function
# I_s = Rm*(Es - V_old)*np.sum(S*g).all()
for it in range(0,40):
I_s += Rm*(Es - V_old)*(S[it]*g[it])
for i in range(0,40):
rand = random.uniform(0,1)
# Spike occures
if rand < Deta_t*r_ini:
S[i] = S[i] + 0.5
# print("Pre_Spike")
# Spikes_Counts[i] += 1
if t <80000: #first 20 seconds
(Pre_Neuron_Spikes_20s[i]).append(t+1)
if t >799999: #last 100 seconds
(Pre_Neuron_Spikes_200s[i]).append(t+1)
if STDP_Mode:
# Store the Pre Spike time
Recent_Pre_Spikes[i] = t
STDP_Deta_t = (Recent_Post_Spike - Recent_Pre_Spikes[i])
# if STDP_Deta_t <= 0:
g[i] = g[i] - A_minus * math.exp(-abs((STDP_Deta_t)*Deta_t)/tau_minus)
if g[i] < 0:
g[i] = 0
# Spike not occures
else:
S[i] = S[i]-S[i]*Deta_t/tau_s
# Update The neuron:
# Post-Synaptic neurons
# I_s = Rm*(Es - V_old)*np.sum(S*g)
V_new = V_old + Deta_t*((EL - V_old + Rm*Ie+I_s) / tau_m)
# If there is a spike and update the V value
if V_new > Vth:
V_new = V_reset
Recent_Post_Spike = t
Spike_Count +=1
if t < 80000:
Post_Neuron_Spikes_20s.append(t+1)
if t > 799999:
Post_Neuron_Spikes_200s.append(t+1)
# print("Post Spike!!")
if STDP_Mode:
for p in range(0,40):
STDP_Deta_t = (Recent_Post_Spike - Recent_Pre_Spikes[i])
# if STDP_Deta_t > 0:
g[p] = g[p] + A_plus * math.exp(-abs((STDP_Deta_t)*Deta_t)/tau_plus)
if g[p] > (4*nS):
g[p] = 4*nS
V_old = V_new
V.append(V_new)
# x, y = np.random.randn(2, 100)
diff_20s = []
diff_200s = []
for index in range(0,40,1):
find = Pre_Neuron_Spikes_20s[index]
for v in find:
for k in Post_Neuron_Spikes_20s:
deta = (v-k)/4.00
if deta < 50 or deta>-50:
diff_20s.append(int(deta))
for index in range(0,40,1):
find = Pre_Neuron_Spikes_200s[index]
for v in find:
for k in Post_Neuron_Spikes_200s:
deta = (v-k)/4.00
if deta < 50 or deta>-50:
diff_200s.append(int(deta))
region = []
for index2 in range(-50,51,1):
region.append(index2)
hist20s, bins = np.histogram(diff_20s,region)
hist200s, bins2 = np.histogram(diff_200s,region)
# for v in Pre_Neuron_Spikes:
# for k in Post_Neuron_Spikes:
# diff.append(int(v-k))
print(len(diff_20s))
print("his: ",(hist20s))
print("BIns: ",len(bins))
X = range(-49,51,1)
title = "Cross-correlogram (STDP: on)"
plt.figure()
plt.bar([i - 0.4 for i in X], (hist20s/N)/hist20s[50],width=0.4,color = 'r',label = "Begin 20s")
plt.bar([i + 0.4 for i in X], (hist200s/N)/hist200s[50],width=0.4,color = 'g',label = "last 100s")
plt.title(title)
plt.legend()
plt.ylabel("coefficient")
plt.xlabel("time/ms")
# plt.savefig('q4_cross_corr_off')
# plt.show()
# plt.title(title)
# plt.legend()
# plt.ylabel("coefficient")
# plt.xlabel("time/ms")
# plt.savefig('q4_cross_corr_off')
plt.show()
def generate_image_from_matrix(puzzle_matrix):
def text(image, digit, row, column):
text_size = cv2.getTextSize(str(digit), font, font_scale,
font_thickness)
text_width = text_size[0][0]
text_height = text_size[0][1]
location = (column * cell_size + border_size + int(text_width * 0.4),
row * cell_size + border_size + int(text_height * 1.4))
cv2.putText(image, str(digit), location, font, font_scale, black,
font_thickness, cv2.LINE_AA)
font_scale = 1.1
font = cv2.FONT_HERSHEY_SIMPLEX
font_thickness = 2
cell_size = 40
border_size = 40
black = (0, 0, 0)
light_gray = (240, 240, 240)
dark_gray = (180, 180, 180)
white = (255, 255, 255)
line_width = 1
image_dimension = cell_size * 9 + border_size * 2
matrix_size = (image_dimension, image_dimension, 3)
matrix_image = np.full(matrix_size, 255, dtype=np.uint8)
cv2.rectangle(matrix_image, (border_size, border_size),
(cell_size * 9 + border_size, cell_size * 9 + border_size),
light_gray, -1)
cv2.rectangle(matrix_image, (cell_size * 3 + border_size, border_size),
(cell_size * 6 + border_size, cell_size * 3 + border_size),
dark_gray, -1)
cv2.rectangle(matrix_image, (border_size, cell_size * 3 + border_size),
(cell_size * 3 + border_size, cell_size * 6 + border_size),
dark_gray, -1)
cv2.rectangle(matrix_image,
(cell_size * 6 + border_size, cell_size * 3 + border_size),
(cell_size * 9 + border_size, cell_size * 6 + border_size),
dark_gray, -1)
cv2.rectangle(matrix_image,
(cell_size * 3 + border_size, cell_size * 6 + border_size),
(cell_size * 6 + border_size, cell_size * 9 + border_size),
dark_gray, -1)
# add the vertical lines
for i in range(0, 10):
if i == 0:
cv2.line(
matrix_image, (i * cell_size + border_size, border_size),
(i * cell_size + border_size, cell_size * 9 + border_size),
black, line_width * 2)
elif i == 9:
cv2.line(matrix_image,
(i * cell_size - line_width + border_size, border_size),
(i * cell_size - line_width + border_size,
cell_size * 9 + border_size), black, line_width * 2)
else:
cv2.line(
matrix_image, (i * cell_size + border_size, border_size),
(i * cell_size + border_size, cell_size * 9 + border_size),
black, line_width * 1)
# add the horizontal lines
for i in range(0, 10):
if i == 0:
cv2.line(
matrix_image, (border_size, i * cell_size + border_size),
(cell_size * 9 + border_size, i * cell_size + border_size),
black, line_width * 2)
elif i == 9:
cv2.line(matrix_image,
(border_size, i * cell_size - line_width + border_size),
(cell_size * 9 + border_size,
i * cell_size - line_width + border_size), black,
line_width * 2)
else:
cv2.line(
matrix_image, (border_size, i * cell_size + border_size),
(cell_size * 9 + border_size, i * cell_size + border_size),
black, line_width * 1)
for row in range(len(puzzle_matrix)):
for column in range(len(puzzle_matrix)):
if puzzle_matrix[row][column] != 0:
text(matrix_image, puzzle_matrix[row][column], row, column)
return matrix_image
def sess_fn(sess):
#sess.run(tf.global_variables_initializer())
return sess.run(
[a, b],
feed_dict={i: np.full((2, 3), 1.0)
for i in [x, y, z]})
def COMSM2127_2(STDP_Mode):
# Pre-Setting
B = 0 # 0Hz
# Units
ms = 0.001
mv = 0.001
MO = 1.0e6
nA = 1.0e-9
nS = 1.0e-9
# Neuron Param
tau_m = 10*ms
EL = -65*mv
V_reset = -65*mv
Vth = -50*mv
Rm = 100*MO
Ie = 0
N = 40
Recent_Post_Spike = -1000
# Synapses
tau_s = 2*ms
gi = 4*nS
ga = 2.08*nS
Es = 0
Deta_s = 0.5
# 40 incoming synapses
S = np.zeros(N)
Spikes_Counts = np.zeros(N)
Recent_Pre_Spikes = np.zeros(N)
Deta_t = 0.25*ms
# r_ini = Inuput_Rate #15 Hz
# SDTP par
A_plus = 0.2*nS
A_minus = 0.25*nS
tau_plus = 20*ms
tau_minus = 20*ms
if STDP_Mode:
g = np.full(N,gi)
else:
g = np.full(N,gi)
# Calculate the Time interval
Runtime = 300
TimeSteps = math.ceil(Runtime/Deta_t)
print("TimeSteps: ", TimeSteps)
V = []
V_old = V_reset
print("The g_i is : ",g)
# Question4_Par
freq = 10 # 10Hz
r_0 = 15 # 20Hz
# Pre-synaptic neurons N:
Spike_Count = 0
# example_pre_index = 17
Pre_Neuron_Spikes_20s = []
Pre_Neuron_Spikes_200s = []
for example in range(0,N,1):
Pre_Neuron_Spikes_20s.append([])
Pre_Neuron_Spikes_200s.append([])
Post_Neuron_Spikes_20s = []
Post_Neuron_Spikes_200s = []
for t in range(0,TimeSteps):
I_s = 0
# <r>(t) = <r>0 + B*sin(2*pai*frequency*t)
r_ini = r_0 + B * math.sin(2*(math.pi)*freq*((t+1)*Deta_t))
# Apply a possion process use random function
# I_s = Rm*(Es - V_old)*np.sum(S*g).all()
for it in range(0,40):
I_s += Rm*(Es - V_old)*(S[it]*g[it])
for i in range(0,40):
rand = random.uniform(0,1)
# Spike occures
if rand < Deta_t*r_ini:
S[i] = S[i] + 0.5
# print("Pre_Spike")
# Spikes_Counts[i] += 1
if t <80000: #first 20 seconds
(Pre_Neuron_Spikes_20s[i]).append(t/4)
if t >799999: #last 100 seconds
(Pre_Neuron_Spikes_200s[i]).append(t/4)
if STDP_Mode:
# Store the Pre Spike time
Recent_Pre_Spikes[i] = t
STDP_Deta_t = (Recent_Post_Spike - Recent_Pre_Spikes[i])
# if STDP_Deta_t <= 0:
g[i] = g[i] - A_minus * math.exp(-abs((STDP_Deta_t)*Deta_t)/tau_minus)
if g[i] < 0:
g[i] = 0
# Spike not occures
else:
S[i] = S[i]-S[i]*Deta_t/tau_s
# Update The neuron:
# Post-Synaptic neurons
# I_s = Rm*(Es - V_old)*np.sum(S*g)
V_new = V_old + Deta_t*((EL - V_old + Rm*Ie+I_s) / tau_m)
# If there is a spike and update the V value
if V_new > Vth:
V_new = V_reset
Recent_Post_Spike = t
Spike_Count +=1
if t < 80000:
Post_Neuron_Spikes_20s.append(t/4)
if t > 799999:
Post_Neuron_Spikes_200s.append(t/4)
# print("Post Spike!!")
if STDP_Mode:
for p in range(0,40):
STDP_Deta_t = (Recent_Post_Spike - Recent_Pre_Spikes[i])
# if STDP_Deta_t > 0:
g[p] = g[p] + A_plus * math.exp(-abs((STDP_Deta_t)*Deta_t)/tau_plus)
if g[p] > (4*nS):
g[p] = 4*nS
V_old = V_new
V.append(V_new)
# x, y = np.random.randn(2, 100)
diff_20s = [0]*101
diff_200s = [0]*101
Post_Neuron_Spikes_20s = np.array(Post_Neuron_Spikes_20s)
Post_Neuron_Spikes_200s = np.array(Post_Neuron_Spikes_200s)
X_axis = np.arange(-50,51,1)
for index in range(0,40,1):
for Target_Spike20 in Pre_Neuron_Spikes_20s[index]:
for X_index in range(1,len(X_axis),1):
shift20 = Target_Spike20 - Post_Neuron_Spikes_20s
count = len(shift20[(shift20>(X_axis[X_index-1])) & (shift20<(X_axis[X_index]))])
diff_20s[X_index-1] +=count
diff_20s = np.array(diff_20s)/N
# plt.figure()
# plt.bar(X_axis,diff_20s,width=0.4,color = 'r',label = "Begin 20s")
# plt.show()
X_axis = np.arange(-50,51,1)
for index in range(0,40,1):
for Target_Spike200 in Pre_Neuron_Spikes_200s[index]:
for X_index in range(1,len(X_axis),1):
shift200 = Target_Spike200 - Post_Neuron_Spikes_200s
count = len(shift200[(shift200>(X_axis[X_index-1])) & (shift200<(X_axis[X_index]))])
diff_200s[X_index-1] +=count
diff_200s = np.array(diff_200s)/N
# plt.figure()
# plt.bar(X_axis,diff_200s,width=0.4,color = 'r',label = "Last 100s")
# plt.show()
return diff_20s,diff_200s
def getDirectedSpread(visibility_matrix: np.ndarray, environment: np.ndarray, resolution: int) -> np.ndarray:
"""
Parameters
----------
visibility_matrix
Holds all visibility maps corresponding to each positions in the environment
environment
Used for finding non zero element when initializing distanceMatrix
directionToTable
0 if table to the right of agent
1 if table down of agent
2 if table to the left of agent
3 if table is up from the agent
resolution
Resolution parameter.
Changes the resolution of the environmental map.
Returns
-------
cone : np.ndarray
Represent a cone shaped vision in the direction to the table.
Has values in [0,1].
This matrix is later multiplied with the particle spread matrix corresponding
to current time step in order to decrease backward spread.
"""
#non_empty_pos = next(zip(*np.where(environment != TileType.WALL & environment != TileType.TABLE)))
#local_width, local_height = visibility_matrix[non_empty_pos].shape
found = False
while found == False:
for j in range(5,np.shape(environment)[0]):
for i in range(5,np.shape(environment)[1]):
if environment[j,i] != 0:
distances = np.full(np.shape(visibility_matrix[int(j),int(i)]), 1, dtype=float)
found = True
local_width, local_height = distances.shape
assert(local_width > 0)
assert(local_height > 0)
radius = 4*resolution
radii = np.linspace(0, radius, 30)
thetas = np.linspace(-np.pi, 0, 40)
cone = np.zeros((radius*2 + 1, radius*2 + 1), dtype=float)
for theta in thetas:
for rad in radii:
# Center of visibility calculation in local coordinates.
local_x = radius
local_y = radius
# 0.5 since radius does not take in account that indexing starts at 0
dx = int(0.5 + np.cos(theta) * rad)
dy = int(np.floor(0.5 + np.sin(theta) * rad)) # np.floor since int() rounds up negative numbers
assert(-radius <= dx <= radius)
assert(-radius <= dy <= radius)
if cone[local_x + dx, local_y + dy] < 0.01:
if np.cos(theta) < -0.75:
cone[local_x + dx, local_y + dy] = 0.1
else:
cone[local_x + dx - 1, local_y + dy] = 5/8 + (3/8 * np.cos(theta))
dx = int(0.5 + np.cos(-theta) * rad)
dy = int(0.5 + np.sin(-theta) * rad)
assert(-radius <= dx <= radius)
assert(-radius <= dy <= radius)
if cone[local_x + dx, local_y + dy] < 0.01:
if np.cos(-theta) < -0.75:
cone[local_x + dx, local_y + dy] = 0.1
else:
cone[local_x + dx - 1, local_y + dy] = 5/8 + (3/8 * np.cos(-theta))
cones = np.zeros((1, 4), dtype=object)
cones[0,0] = cone
cones[0,1] = np.rot90(cone)
cones[0,2] = np.rot90(cone,2)
cones[0,3] = np.rot90(cone,3)
return cones
def Question_1():
# PartB Spike-timing-dependent plasticity
# Units
ms = 0.001
mv = 0.001
MO = 1.0e6
nA = 1.0e-9
nS = 1.0e-9
# Neuron Param
tau_m = 10*ms
EL = -65*mv
V_reset = -65*mv
Vth = -50*mv
Rm = 100*MO
Ie = 0
N = 40
# Synapses
tau_s = 2*ms
gi = 4*nS
Es = 0
Deta_s = 0.5
# 40 incoming synapses
S = np.zeros((1,40))
g = np.full((1,40),gi)
Deta_t = 0.25*ms
r_ini = 15 #15 Hz
# Euler's method
print("The g_i is : ",g)
# Calculate the Time interval
TimeSteps = math.ceil(1/Deta_t)
# print("TimeSteps: ", TimeSteps)
V = []
V_old = V_reset
Count = 0
S0 = S
for _ in range(0,TimeSteps):
I_s = Rm*(Es - V_old)*np.sum(S0*gi)
# print(I_s)
V_new = V_old + Deta_t*((EL - V_old + Rm*Ie+I_s) / tau_m)
Possiond = Possion(Deta_t,r_ini)
# S = S0+S0*Deta_t/tau_s
# Apply a possion process use random function
for i in range(0,40):
rand = random.uniform(0,1)
if rand < Deta_t*r_ini:
S[0,i] = S0[0,i] + 0.5
else:
S[0,i] = S0[0,i] - S0[0,i]*Deta_t/tau_s
# If there is a spike and update the V value
if V_new > Vth:
V_new = V_reset
Count+=1
V_old = V_new
S0 = S
V.append(V_new)
print("The fir rate is: ",Count)
plt.figure()
plt.plot(range(0,TimeSteps),V,'g')
plt.title("40 synapses with fixed $\overline {g_i}$ in 1ms")
plt.xlabel("Time/0.25 ms total 1s wiht 0.25ms intervals")
plt.ylabel("Voltage/mv")
plt.savefig('./coursework3/graphs/PartB_Question1.png')
def test_incremental_variance_numerical_stability():
# Test Youngs and Cramer incremental variance formulas.
def np_var(A):
return A.var(axis=0)
# Naive one pass variance computation - not numerically stable
# https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
def one_pass_var(X):
n = X.shape[0]
exp_x2 = (X**2).sum(axis=0) / n
expx_2 = (X.sum(axis=0) / n)**2
return exp_x2 - expx_2
# Two-pass algorithm, stable.
# We use it as a benchmark. It is not an online algorithm
# https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass_algorithm
def two_pass_var(X):
mean = X.mean(axis=0)
Y = X.copy()
return np.mean((Y - mean)**2, axis=0)
# Naive online implementation
# https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm
# This works only for chunks for size 1
def naive_mean_variance_update(x, last_mean, last_variance,
last_sample_count):
updated_sample_count = (last_sample_count + 1)
samples_ratio = last_sample_count / float(updated_sample_count)
updated_mean = x / updated_sample_count + last_mean * samples_ratio
updated_variance = last_variance * samples_ratio + \
(x - last_mean) * (x - updated_mean) / updated_sample_count
return updated_mean, updated_variance, updated_sample_count
# We want to show a case when one_pass_var has error > 1e-3 while
# _batch_mean_variance_update has less.
tol = 200
n_features = 2
n_samples = 10000
x1 = np.array(1e8, dtype=np.float64)
x2 = np.log(1e-5, dtype=np.float64)
A0 = np.full((n_samples // 2, n_features), x1, dtype=np.float64)
A1 = np.full((n_samples // 2, n_features), x2, dtype=np.float64)
A = np.vstack((A0, A1))
# Older versions of numpy have different precision
# In some old version, np.var is not stable
if np.abs(np_var(A) - two_pass_var(A)).max() < 1e-6:
stable_var = np_var
else:
stable_var = two_pass_var
# Naive one pass var: >tol (=1063)
assert_greater(np.abs(stable_var(A) - one_pass_var(A)).max(), tol)
# Starting point for online algorithms: after A0
# Naive implementation: >tol (436)
mean, var, n = A0[0, :], np.zeros(n_features), n_samples // 2
for i in range(A1.shape[0]):
mean, var, n = \
naive_mean_variance_update(A1[i, :], mean, var, n)
assert_equal(n, A.shape[0])
# the mean is also slightly unstable
assert_greater(np.abs(A.mean(axis=0) - mean).max(), 1e-6)
assert_greater(np.abs(stable_var(A) - var).max(), tol)
# Robust implementation: <tol (177)
mean, var = A0[0, :], np.zeros(n_features)
n = np.full(n_features, n_samples // 2, dtype=np.int32)
for i in range(A1.shape[0]):
mean, var, n = \
_incremental_mean_and_var(A1[i, :].reshape((1, A1.shape[1])),
mean, var, n)
assert_array_equal(n, A.shape[0])
assert_array_almost_equal(A.mean(axis=0), mean)
assert_greater(tol, np.abs(stable_var(A) - var).max())
def check_with_place(self, place):
scope = core.Scope()
# create and initialize Grad Variable
height = 10
rows = [0, 4, 7]
row_numel = 12
mu = 1.0
use_nesterov = self.use_nesterov
regularization_method = self.regularization_method
regularization_coeff = self.regularization_coeff
# create and initialize Param Variable
param_array = np.full((height, row_numel), 5.0).astype("float32")
param_out_array = np.full((height, row_numel), 0.0).astype("float32")
param = scope.var('Param').get_tensor()
param.set(param_array.astype("float16"), place)
param_out = scope.var("ParamOut").get_tensor()
param_out.set(param_out_array.astype("float16"), place)
master_param = scope.var('MasterParam').get_tensor()
master_param.set(param_array, place)
master_param_out = scope.var("MasterParamOut").get_tensor()
master_param_out.set(param_out_array, place)
grad_selected_rows = scope.var('Grad').get_selected_rows()
grad_selected_rows.set_height(height)
grad_selected_rows.set_rows(rows)
grad_np_array = np.ones((len(rows), row_numel)).astype("float32")
grad_np_array[0, 0] = 2.0
grad_np_array[2, 8] = 4.0
grad_tensor = grad_selected_rows.get_tensor()
grad_tensor.set(grad_np_array.astype("float16"), place)
velocity = scope.var('Velocity').get_tensor()
velocity_np_array = np.ones((height, row_numel)).astype("float32")
velocity.set(velocity_np_array, place)
velocity_out = scope.var('VelocityOut').get_tensor()
velocity_out_np_array = np.full((height, row_numel),
0.0).astype("float32")
velocity_out.set(velocity_out_np_array, place)
# create and initialize LearningRate Variable
lr = scope.var('LearningRate').get_tensor()
lr_array = np.full((1), 2.0).astype("float32")
lr.set(lr_array, place)
# create and run operator
op = Operator(
"momentum",
Param='Param',
Grad='Grad',
Velocity='Velocity',
MasterParam='MasterParam',
ParamOut='ParamOut',
VelocityOut='VelocityOut',
MasterParamOut='MasterParamOut',
LearningRate='LearningRate',
mu=mu,
use_nesterov=use_nesterov,
regularization_method=regularization_method,
regularization_coeff=regularization_coeff,
multi_precision=True,
rescale_grad=1.0)
op.run(scope, place)
# get and compare result
param_out_np_array = np.array(param_out)
velocity_out_np_array = np.array(velocity_out)
_grad_np_array = np.full((height, row_numel), 0.0).astype("float32")
for i in range(len(rows)):
_grad_np_array[rows[i]] = grad_np_array[i]
_param = param_array
_param_out, _velocity_out = calculate_momentum_by_numpy(
param=_param,
grad=_grad_np_array,
mu=mu,
velocity=velocity_np_array,
use_nesterov=use_nesterov,
learning_rate=lr_array,
regularization_method=regularization_method,
regularization_coeff=regularization_coeff)
self.assertTrue((_velocity_out == velocity_out_np_array).all())
self.assertTrue((_param_out == param_out_np_array).all())
def get_matrix_data(
self, label, buffer_manager, region, placements, graph_mapper,
application_vertex, variable, n_machine_time_steps):
""" Read a uint32 mapped to time and neuron IDs from the SpiNNaker\
machine.
:param label: vertex label
:param buffer_manager: the manager for buffered data
:param region: the DSG region ID used for this data
:param placements: the placements object
:param graph_mapper: \
the mapping between application and machine vertices
:param application_vertex:
:param variable: PyNN name for the variable (V, gsy_inh etc.)
:type variable: str
:param n_machine_time_steps:
:return:
"""
if variable == SPIKES:
msg = "Variable {} is not supported use get_spikes".format(SPIKES)
raise ConfigurationException(msg)
vertices = graph_mapper.get_machine_vertices(application_vertex)
progress = ProgressBar(
vertices, "Getting {} for {}".format(variable, label))
sampling_rate = self._sampling_rates[variable]
expected_rows = int(math.ceil(
n_machine_time_steps / sampling_rate))
missing_str = ""
data = None
indexes = []
for vertex in progress.over(vertices):
placement = placements.get_placement_of_vertex(vertex)
vertex_slice = graph_mapper.get_slice(vertex)
neurons = self._neurons_recording(variable, vertex_slice)
n_neurons = len(neurons)
if n_neurons == 0:
continue
indexes.extend(neurons)
# for buffering output info is taken form the buffer manager
neuron_param_region_data_pointer, missing_data = \
buffer_manager.get_data_for_vertex(
placement, region)
record_raw = neuron_param_region_data_pointer.read_all()
record_length = len(record_raw)
row_length = self.N_BYTES_FOR_TIMESTAMP + \
n_neurons * self.N_BYTES_PER_VALUE
# There is one column for time and one for each neuron recording
n_rows = record_length // row_length
# Converts bytes to ints and make a matrix
record = (numpy.asarray(record_raw, dtype="uint8").
view(dtype="<i4")).reshape((n_rows, (n_neurons + 1)))
# Check if you have the expected data
if not missing_data and n_rows == expected_rows:
# Just cut the timestamps off to get the fragment
fragment = (record[:, 1:] / float(DataType.S1615.scale))
else:
missing_str += "({}, {}, {}); ".format(
placement.x, placement.y, placement.p)
# Start the fragment for this slice empty
fragment = numpy.empty((expected_rows, n_neurons))
for i in xrange(0, expected_rows):
time = i * sampling_rate
# Check if there is data for this timestep
local_indexes = numpy.where(record[:, 0] == time)
if len(local_indexes[0]) > 0:
# Set row to data for that timestep
fragment[i] = (record[local_indexes[0], 1:] /
float(DataType.S1615.scale))
else:
# Set row to nan
fragment[i] = numpy.full(n_neurons, numpy.nan)
if data is None:
data = fragment
else:
# Add the slice fragment on axis 1 which is IDs/channel_index
data = numpy.append(data, fragment, axis=1)
if len(missing_str) > 0:
logger.warn(
"Population {} is missing recorded data in region {} from the"
" following cores: {}".format(label, region, missing_str))
sampling_interval = self.get_neuron_sampling_interval(variable)
return (data, indexes, sampling_interval)
def Stochastic_Process(j):
Profile, num_profiles = Initialise_model()
peak_enlarg, mu_peak, s_peak, Year_behaviour, User_list = Initialise_inputs(j)
'''
Calculation of the peak time range, which is used to discriminate between off-peak and on-peak coincident switch-on probability
Calculates first the overall Peak Window (taking into account all User classes).
The peak window is just a time window in which coincident switch-on of multiple appliances assumes a higher probability than off-peak
Within the peak window, a random peak time is calculated and then enlarged into a peak_time_range following again a random procedure
'''
windows_curve = np.zeros(1440) #creates an empty daily profile
Tot_curve = np.zeros(1440) #creates another empty daily profile
for Us in User_list:
App_count = 0
for App in Us.App_list:
#Calculate windows curve, i.e. the theoretical maximum curve that can be obtained, for each app, by switching-on always all the 'n' apps altogether in any time-step of the functioning windows
single_wcurve = Us.App_list[App_count].daily_use*np.mean(Us.App_list[App_count].POWER)*Us.App_list[App_count].number #this computes the curve for the specific App
windows_curve = np.vstack([windows_curve, single_wcurve]) #this stacks the specific App curve in an overall curve comprising all the Apps within a User class
App_count += 1
Us.windows_curve = windows_curve #after having iterated for all the Apps within a User class, saves the overall User class theoretical maximum curve
Us.windows_curve = np.transpose(np.sum(Us.windows_curve, axis = 0))*Us.num_users
Tot_curve = Tot_curve + Us.windows_curve #adds the User's theoretical max profile to the total theoretical max comprising all classes
peak_window = np.transpose(np.argwhere(Tot_curve == np.amax(Tot_curve))) #Find the peak window within the theoretical max profile
peak_time = round(random.normalvariate(round(np.average(peak_window)),1/3*(peak_window[0,-1]-peak_window[0,0]))) #Within the peak_window, randomly calculate the peak_time using a gaussian distribution
peak_time_range = np.arange((peak_time-round(math.fabs(peak_time-(random.gauss(peak_time,(peak_enlarg*peak_time)))))),(peak_time+round(math.fabs(peak_time-random.gauss(peak_time,(peak_enlarg*peak_time)))))) #the peak_time is randomly enlarged based on the calibration parameter peak_enlarg
'''
The core stochastic process starts here. For each profile requested by the software user,
each Appliance instance within each User instance is separately and stochastically generated
'''
for prof_i in range(num_profiles): #the whole code is repeated for each profile that needs to be generated
Tot_Classes = np.zeros(1440) #initialise an empty daily profile that will be filled with the sum of the hourly profiles of each User instance
for Us in User_list: #iterates for each User instance (i.e. for each user class)
Us.load = np.zeros(1440) #initialise empty load for User instance
for i in range(Us.num_users): #iterates for every single user within a User class. Each single user has its own separate randomisation
if Us.user_preference == 0:
rand_daily_pref = 0
pass
else:
rand_daily_pref = random.randint(1,Us.user_preference)
for App in Us.App_list: #iterates for all the App types in the given User class
#initialises variables for the cycle
tot_time = 0
App.daily_use = np.zeros(1440)
if random.uniform(0,1) > App.occasional_use: #evaluates if occasional use happens or not
continue
else:
pass
if App.Pref_index == 0:
pass
else:
if rand_daily_pref == App.Pref_index: #evaluates if daily preference coincides with the randomised daily preference number
pass
else:
continue
if App.wd_we == Year_behaviour[prof_i] or App.wd_we == 2 : #checks if the app is allowed in the given yearly behaviour pattern
pass
else:
continue
#recalculate windows start and ending times randomly, based on the inputs
rand_window_1 = np.array([int(random.uniform((App.window_1[0]-App.random_var_1),(App.window_1[0]+App.random_var_1))),int(random.uniform((App.window_1[1]-App.random_var_1),(App.window_1[1]+App.random_var_1)))])
if rand_window_1[0] < 0:
rand_window_1[0] = 0
if rand_window_1[1] > 1440:
rand_window_1[1] = 1440
rand_window_2 = np.array([int(random.uniform((App.window_2[0]-App.random_var_2),(App.window_2[0]+App.random_var_2))),int(random.uniform((App.window_2[1]-App.random_var_2),(App.window_2[1]+App.random_var_2)))])
if rand_window_2[0] < 0:
rand_window_2[0] = 0
if rand_window_2[1] > 1440:
rand_window_2[1] = 1440
rand_window_3 = np.array([int(random.uniform((App.window_3[0]-App.random_var_3),(App.window_3[0]+App.random_var_3))),int(random.uniform((App.window_3[1]-App.random_var_3),(App.window_3[1]+App.random_var_3)))])
if rand_window_3[0] < 0:
rand_window_3[0] = 0
if rand_window_3[1] > 1440:
rand_window_3[1] = 1440
#redefines functioning windows based on the previous randomisation of the boundaries
if App.flat == 'yes': #if the app is "flat" the code stops right after filling the newly created windows without applying any further stochasticity
App.daily_use[rand_window_1[0]:rand_window_1[1]] = np.full(np.diff(rand_window_1),App.POWER[prof_i]*App.number)
App.daily_use[rand_window_2[0]:rand_window_2[1]] = np.full(np.diff(rand_window_2),App.POWER[prof_i]*App.number)
App.daily_use[rand_window_3[0]:rand_window_3[1]] = np.full(np.diff(rand_window_3),App.POWER[prof_i]*App.number)
Us.load = Us.load + App.daily_use
continue
else: #otherwise, for "non-flat" apps it puts a mask on the newly defined windows and continues
App.daily_use[rand_window_1[0]:rand_window_1[1]] = np.full(np.diff(rand_window_1),0.001)
App.daily_use[rand_window_2[0]:rand_window_2[1]] = np.full(np.diff(rand_window_2),0.001)
App.daily_use[rand_window_3[0]:rand_window_3[1]] = np.full(np.diff(rand_window_3),0.001)
App.daily_use_masked = np.zeros_like(ma.masked_not_equal(App.daily_use,0.001))
App.power = App.POWER[prof_i]
#random variability is applied to the total functioning time and to the duration of the duty cycles, if they have been specified
random_var_t = random.uniform((1-App.r_t),(1+App.r_t))
if App.activate == 1:
App.p_11 = App.P_11*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_12 = App.P_12*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
random_cycle1 = np.concatenate(((np.ones(int(App.t_11*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_11),(np.ones(int(App.t_12*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_12))) #randomise also the fixed cycle
random_cycle2 = random_cycle1
random_cycle3 = random_cycle1
elif App.activate == 2:
App.p_11 = App.P_11*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_12 = App.P_12*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_21 = App.P_21*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_22 = App.P_22*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
random_cycle1 = np.concatenate(((np.ones(int(App.t_11*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_11),(np.ones(int(App.t_12*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_12))) #randomise also the fixed cycle
random_cycle2 = np.concatenate(((np.ones(int(App.t_21*(random.uniform((1+App.r_c2),(1-App.r_c2)))))*App.p_21),(np.ones(int(App.t_22*(random.uniform((1+App.r_c2),(1-App.r_c2)))))*App.p_22))) #randomise also the fixed cycle
random_cycle3 = random_cycle1
elif App.activate == 3:
App.p_11 = App.P_11*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_12 = App.P_12*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_21 = App.P_12*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_22 = App.P_22*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_31 = App.P_31*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
App.p_32 = App.P_32*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var))) #randomly variates the power of thermal apps, otherwise variability is 0
random_cycle1 = random.choice([np.concatenate(((np.ones(int(App.t_11*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_11),(np.ones(int(App.t_12*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_12))),np.concatenate(((np.ones(int(App.t_12*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_12),(np.ones(int(App.t_11*(random.uniform((1+App.r_c1),(1-App.r_c1)))))*App.p_11)))]) #randomise also the fixed cycle
random_cycle2 = random.choice([np.concatenate(((np.ones(int(App.t_21*(random.uniform((1+App.r_c2),(1-App.r_c2)))))*App.p_21),(np.ones(int(App.t_22*(random.uniform((1+App.r_c2),(1-App.r_c2)))))*App.p_22))),np.concatenate(((np.ones(int(App.t_22*(random.uniform((1+App.r_c2),(1-App.r_c2)))))*App.p_22),(np.ones(int(App.t_21*(random.uniform((1+App.r_c2),(1-App.r_c2)))))*App.p_21)))])
random_cycle3 = random.choice([np.concatenate(((np.ones(int(App.t_31*(random.uniform((1+App.r_c3),(1-App.r_c3)))))*App.p_31),(np.ones(int(App.t_32*(random.uniform((1+App.r_c3),(1-App.r_c3)))))*App.p_32))),np.concatenate(((np.ones(int(App.t_32*(random.uniform((1+App.r_c3),(1-App.r_c3)))))*App.p_32),(np.ones(int(App.t_31*(random.uniform((1+App.r_c3),(1-App.r_c3)))))*App.p_31)))])#this is to avoid that all cycles are sincronous
else:
pass
rand_time = round(random.uniform(App.func_time,int(App.func_time*random_var_t)))
#control to check that the total randomised time of use does not exceed the total space available in the windows
if rand_time > 0.99*(np.diff(rand_window_1)+np.diff(rand_window_2)+np.diff(rand_window_3)):
rand_time = int(0.99*(np.diff(rand_window_1)+np.diff(rand_window_2)+np.diff(rand_window_3)))
max_free_spot = rand_time #free spots are used to detect if there's still space for switch_ons. Before calculating actual free spots, the max free spot is set equal to the entire randomised func_time
while tot_time <= rand_time: #this is the key cycle, which runs for each App until the switch_ons and their duration equals the randomised total time of use of the App
#check how many windows to consider
if App.num_windows == 1:
switch_on = int(random.choice([random.uniform(rand_window_1[0],(rand_window_1[1]))]))
elif App.num_windows == 2:
switch_on = int(random.choice([random.uniform(rand_window_1[0],(rand_window_1[1])),random.uniform(rand_window_2[0],(rand_window_2[1]))]))
else:
switch_on = int(random.choice([random.uniform(rand_window_1[0],(rand_window_1[1])),random.uniform(rand_window_2[0],(rand_window_2[1])),random.uniform(rand_window_3[0],(rand_window_3[1]))]))
#Identifies a random switch on time within the available functioning windows
if App.daily_use[switch_on] == 0.001: #control to check if the app is not already on at the randomly selected switch-on time
if switch_on in range(rand_window_1[0],rand_window_1[1]):
if np.any(App.daily_use[switch_on:rand_window_1[1]]!=0.001): #control to check if there are any other switch on times after the current one
next_switch = [switch_on + k[0] for k in np.where(App.daily_use[switch_on:]!=0.001)] #identifies the position of next switch on time and sets it as a limit for the duration of the current switch on
if (next_switch[0] - switch_on) >= App.func_cycle and max_free_spot >= App.func_cycle:
upper_limit = min((next_switch[0]-switch_on),min(rand_time,rand_window_1[1]-switch_on))
elif (next_switch[0] - switch_on) < App.func_cycle and max_free_spot >= App.func_cycle: #if next switch_on event does not allow for a minimum functioning cycle without overlapping, but there are other larger free spots, the cycle tries again from the beginning
continue
else:
upper_limit = next_switch[0]-switch_on #if there are no other options to reach the total time of use, empty spaces are filled without minimum cycle restrictions until reaching the limit
else:
upper_limit = min(rand_time,rand_window_1[1]-switch_on) #if there are no other switch-on events after the current one, the upper duration limit is set this way
if upper_limit >= App.func_cycle: #if the upper limit is higher than minimum functioning time, an array of indexes is created to be later put in the profile
indexes = np.arange(switch_on,switch_on+(int(random.uniform(App.func_cycle,upper_limit)))) #a random duration is chosen between the upper limit and the minimum cycle
else:
indexes = np.arange(switch_on,switch_on+upper_limit) #this is the case in which empty spaces need to be filled without constraints to reach the total time goal
elif switch_on in range(rand_window_2[0],rand_window_2[1]): #if random switch_on happens in windows2, same code as above is repeated for windows2
if np.any(App.daily_use[switch_on:rand_window_2[1]]!=0.001):
next_switch = [switch_on + k[0] for k in np.where(App.daily_use[switch_on:]!=0.001)]
if (next_switch[0] - switch_on) >= App.func_cycle and max_free_spot >= App.func_cycle:
upper_limit = min((next_switch[0]-switch_on),min(rand_time,rand_window_2[1]-switch_on))
elif (next_switch[0] - switch_on) < App.func_cycle and max_free_spot >= App.func_cycle:
continue
else:
upper_limit = next_switch[0]-switch_on
else:
upper_limit = min(rand_time,rand_window_2[1]-switch_on)
if upper_limit >= App.func_cycle:
indexes = np.arange(switch_on,switch_on+(int(random.uniform(App.func_cycle,upper_limit))))
else:
indexes = np.arange(switch_on,switch_on+upper_limit)
else: #if switch_on is not in window1 nor in window2, it shall be in window3. Same code is repreated
if np.any(App.daily_use[switch_on:rand_window_3[1]]!=0.001):
next_switch = [switch_on + k[0] for k in np.where(App.daily_use[switch_on:]!=0.001)]
if (next_switch[0] - switch_on) >= App.func_cycle and max_free_spot >= App.func_cycle:
upper_limit = min((next_switch[0]-switch_on),min(rand_time,rand_window_3[1]-switch_on))
elif (next_switch[0] - switch_on) < App.func_cycle and max_free_spot >= App.func_cycle:
continue
else:
upper_limit = next_switch[0]-switch_on
else:
upper_limit = min(rand_time,rand_window_3[1]-switch_on)
if upper_limit >= App.func_cycle:
indexes = np.arange(switch_on,switch_on+(int(random.uniform(App.func_cycle,upper_limit))))
else:
indexes = np.arange(switch_on,switch_on+upper_limit)
tot_time = tot_time + indexes.size #the count of total time is updated with the size of the indexes array
if tot_time > rand_time: #control to check when the total functioning time is reached. It will be typically overcome, so a correction is applied to avoid this
indexes_adj = indexes[:-(tot_time-rand_time)] #correctes indexes size to avoid overcoming total time
if np.in1d(peak_time_range,indexes_adj).any() and App.fixed == 'no': #check if indexes are in peak window and if the coincident behaviour is locked by the "fixed" attribute
coincidence = min(App.number,max(1,math.ceil(random.gauss(math.ceil(App.number*mu_peak),(s_peak*App.number*mu_peak))))) #calculates coincident behaviour within the peak time range
elif np.in1d(peak_time_range,indexes_adj).any()== False and App.fixed == 'no': #check if indexes are off-peak and if coincident behaviour is locked or not
Prob = random.uniform(0,(App.number-1)/App.number) #calculates probability of coincident switch_ons off-peak
array = np.arange(0,App.number)/App.number
try:
on_number = np.max(np.where(Prob>=array))+1
except ValueError:
on_number = 1
coincidence = on_number #randomly selects how many apps are on at the same time for each app type based on the above probabilistic algorithm
else:
coincidence = App.number #this is the case when App.fixed is activated. All 'n' apps of an App instance are switched_on altogether
if App.activate > 0: #evaluates if the app has some duty cycles to be considered
if indexes_adj.size > 0:
evaluate = round(np.mean(indexes_adj)) #calculates the mean time position of the current switch_on event, to later select the proper duty cycle
else:
evaluate = 0
#based on the evaluate value, selects the proper duty cycle and puts the corresponding power values in the indexes range
if evaluate in range(App.cw11[0],App.cw11[1]) or evaluate in range(App.cw12[0],App.cw12[1]):
np.put(App.daily_use,indexes_adj,(random_cycle1*coincidence))
np.put(App.daily_use_masked,indexes_adj,(random_cycle1*coincidence),mode='clip')
elif evaluate in range(App.cw21[0],App.cw21[1]) or evaluate in range(App.cw22[0],App.cw22[1]):
np.put(App.daily_use,indexes_adj,(random_cycle2*coincidence))
np.put(App.daily_use_masked,indexes_adj,(random_cycle2*coincidence),mode='clip')
else:
np.put(App.daily_use,indexes_adj,(random_cycle3*coincidence))
np.put(App.daily_use_masked,indexes_adj,(random_cycle3*coincidence),mode='clip')
else: #if no duty cycles are specififed, a regular switch_on event is modelled
np.put(App.daily_use,indexes_adj,(App.power*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var)))*coincidence)) #randomises also the App Power if Thermal_P_var is on
np.put(App.daily_use_masked,indexes_adj,(App.power*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var)))*coincidence),mode='clip')
App.daily_use_masked = np.zeros_like(ma.masked_greater_equal(App.daily_use_masked,0.001)) #updates the mask excluding the current switch_on event to identify the free_spots for the next iteration
tot_time = (tot_time - indexes.size) + indexes_adj.size #updates the total time correcting the previous value
break #exit cycle and go to next App
else: #if the tot_time has not yet exceeded the App total functioning time, the cycle does the same without applying corrections to indexes size
if np.in1d(peak_time_range,indexes).any() and App.fixed == 'no':
coincidence = min(App.number,max(1,math.ceil(random.gauss(math.ceil(App.number*mu_peak),(s_peak*App.number*mu_peak)))))
elif np.in1d(peak_time_range,indexes).any() == False and App.fixed == 'no':
Prob = random.uniform(0,(App.number-1)/App.number)
array = np.arange(0,App.number)/App.number
try:
on_number = np.max(np.where(Prob>=array))+1
except ValueError:
on_number = 1
coincidence = on_number
else:
coincidence = App.number
if App.activate > 0:
if indexes.size > 0:
evaluate = round(np.mean(indexes))
else:
evaluate = 0
if evaluate in range(App.cw11[0],App.cw11[1]) or evaluate in range(App.cw12[0],App.cw12[1]):
np.put(App.daily_use,indexes,(random_cycle1*coincidence))
np.put(App.daily_use_masked,indexes,(random_cycle1*coincidence),mode='clip')
elif evaluate in range(App.cw21[0],App.cw21[1]) or evaluate in range(App.cw22[0],App.cw22[1]):
np.put(App.daily_use,indexes,(random_cycle2*coincidence))
np.put(App.daily_use_masked,indexes,(random_cycle2*coincidence),mode='clip')
else:
np.put(App.daily_use,indexes,(random_cycle3*coincidence))
np.put(App.daily_use_masked,indexes,(random_cycle3*coincidence),mode='clip')
else:
np.put(App.daily_use,indexes,(App.power*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var)))*coincidence))
np.put(App.daily_use_masked,indexes,(App.power*(random.uniform((1-App.Thermal_P_var),(1+App.Thermal_P_var)))*coincidence),mode='clip')
App.daily_use_masked = np.zeros_like(ma.masked_greater_equal(App.daily_use_masked,0.001))
tot_time = tot_time #no correction applied to previously calculated value
free_spots = [] #calculate how many free spots remain for further switch_ons
try:
for j in ma.notmasked_contiguous(App.daily_use_masked):
free_spots.append(j.stop-j.start)
except TypeError:
free_spots = [0]
max_free_spot = max(free_spots)
else:
continue #if the random switch_on falls somewhere where the App has been already turned on, tries again from beginning of the while cycle
Us.load = Us.load + App.daily_use #adds the App profile to the User load
Tot_Classes = Tot_Classes + Us.load #adds the User load to the total load of all User classes
Profile.append(Tot_Classes) #appends the total load to the list that will contain all the generated profiles
print('Profile',prof_i+1,'/',num_profiles,'completed') #screen update about progress of computation
return(Profile)
def align_name_scores_with_annots(annot_score_list, annot_aid_list, daid2_idx, name_groupxs, name_score_list):
r"""
takes name scores and gives them to the best annotation
Returns:
score_list: list of scores aligned with cm.daid_list and cm.dnid_list
Args:
annot_score_list (list): score associated with each annot
name_groupxs (list): groups annot_score lists into groups compatible with name_score_list
name_score_list (list): score assocated with name
nid2_nidx (dict): mapping from nids to index in name score list
CommandLine:
python -m ibeis.algo.hots.name_scoring --test-align_name_scores_with_annots
python -m ibeis.algo.hots.name_scoring --test-align_name_scores_with_annots --show
Example:
>>> # ENABLE_DOCTEST
>>> from ibeis.algo.hots.name_scoring import * # NOQA
>>> ibs, qreq_, cm_list = plh.testdata_post_sver('PZ_MTEST', qaid_list=[18])
>>> cm = cm_list[0]
>>> cm.evaluate_csum_annot_score(qreq_)
>>> cm.evaluate_nsum_name_score(qreq_)
>>> # Annot aligned lists
>>> annot_score_list = cm.algo_annot_scores['csum']
>>> annot_aid_list = cm.daid_list
>>> daid2_idx = cm.daid2_idx
>>> # Name aligned lists
>>> name_score_list = cm.algo_name_scores['nsum']
>>> name_groupxs = cm.name_groupxs
>>> # Execute Function
>>> score_list = align_name_scores_with_annots(annot_score_list, annot_aid_list, daid2_idx, name_groupxs, name_score_list)
>>> # Check that the correct name gets the highest score
>>> target = name_score_list[cm.nid2_nidx[cm.qnid]]
>>> test_index = np.where(score_list == target)[0][0]
>>> cm.score_list = score_list
>>> ut.assert_eq(ibs.get_annot_name_rowids(cm.daid_list[test_index]), cm.qnid)
>>> assert ut.isunique(cm.dnid_list[score_list > 0]), 'bad name score'
>>> top_idx = cm.algo_name_scores['nsum'].argmax()
>>> assert cm.get_top_nids()[0] == cm.unique_nids[top_idx], 'bug in alignment'
>>> ut.quit_if_noshow()
>>> cm.show_ranked_matches(qreq_)
>>> ut.show_if_requested()
Example:
>>> # DISABLE_DOCTEST
>>> from ibeis.algo.hots.name_scoring import * # NOQA
>>> annot_score_list = []
>>> annot_aid_list = []
>>> daid2_idx = {}
>>> # Name aligned lists
>>> name_score_list = np.array([], dtype=np.float32)
>>> name_groupxs = []
>>> # Execute Function
>>> score_list = align_name_scores_with_annots(annot_score_list, annot_aid_list, daid2_idx, name_groupxs, name_score_list)
"""
if len(name_groupxs) == 0:
score_list = np.empty(0, dtype=name_score_list.dtype)
return score_list
else:
# Group annot aligned indicies by nid
annot_aid_list = np.array(annot_aid_list)
#nid_list, groupxs = vt.group_indices(annot_nid_list)
grouped_scores = vt.apply_grouping(annot_score_list, name_groupxs)
grouped_annot_aids = vt.apply_grouping(annot_aid_list, name_groupxs)
flat_grouped_aids = np.hstack(grouped_annot_aids)
#flat_groupxs = np.hstack(name_groupxs)
#if __debug__:
# sum_scores = np.array([scores.sum() for scores in grouped_scores])
# max_scores = np.array([scores.max() for scores in grouped_scores])
# assert np.all(name_score_list <= sum_scores)
# assert np.all(name_score_list > max_scores)
# +------------
# Find the position of the highest name_scoring annotation for each name
# IN THE FLATTENED GROUPED ANNOT_AID_LIST (this was the bug)
offset_list = np.array([annot_scores.argmax() for annot_scores in grouped_scores])
# Find the starting position of eatch group use chain to start offsets with 0
_padded_scores = itertools.chain([[]], grouped_scores[:-1])
sizeoffset_list = np.array([len(annot_scores) for annot_scores in _padded_scores])
baseindex_list = sizeoffset_list.cumsum()
# Augment starting position with offset index
annot_idx_list = np.add(baseindex_list, offset_list)
# L______________
best_aid_list = flat_grouped_aids[annot_idx_list]
best_idx_list = ut.dict_take(daid2_idx, best_aid_list)
# give the annotation domain a name score
#score_list = np.zeros(len(annot_score_list), dtype=name_score_list.dtype)
score_list = np.full(len(annot_score_list), fill_value=-np.inf,
dtype=name_score_list.dtype)
#score_list = np.full(len(annot_score_list), fill_value=np.nan, dtype=name_score_list.dtype)
#score_list = np.nan(len(annot_score_list), dtype=name_score_list.dtype)
# HACK: we need to set these to 'low' values and we also have to respect negatives
#score_list[:] = -np.inf
# make sure that the nid_list from group_indicies and the nids belonging to
# name_score_list (cm.unique_nids) are in alignment
#nidx_list = np.array(ut.dict_take(nid2_nidx, nid_list))
# THIS ASSUMES name_score_list IS IN ALIGNMENT WITH BOTH cm.unique_nids and
# nid_list (which should be == cm.unique_nids)
score_list[best_idx_list] = name_score_list
return score_list
def _calc_transport_wilcock_crowe(self):
"""Method to determine the transport time for each parcel in the active
layer using a sediment transport equation.
Note: could have options here (e.g. Wilcock and Crowe, FLVB, MPM, etc)
"""
# Initialize _pvelocity, the virtual velocity of each parcel (link length / link travel time)
self._pvelocity = np.zeros(self._num_parcels)
# parcel attribute arrays from DataRecord
Darray = self._parcels.dataset.D[:, self._time_idx]
Activearray = self._parcels.dataset.active_layer[:, self._time_idx].values
Rhoarray = self._parcels.dataset.density.values
Volarray = self._parcels.dataset.volume[:, self._time_idx].values
Linkarray = self._parcels.dataset.element_id[
:, self._time_idx
].values # link that the parcel is currently in
R = (Rhoarray - self._fluid_density) / self._fluid_density
# parcel attribute arrays to populate below
frac_sand_array = np.zeros(self._num_parcels)
vol_act_array = np.zeros(self._num_parcels)
Sarray = np.zeros(self._num_parcels)
Harray = np.zeros(self._num_parcels)
Larray = np.zeros(self._num_parcels)
# D_mean_activearray = np.zeros(self._num_parcels) * (np.nan)
# active_layer_thickness_array = np.zeros(self._num_parcels) * np.nan
D_mean_activearray = np.full(self._num_parcels, np.nan)
active_layer_thickness_array = np.full(self._num_parcels, np.nan)
# rhos_mean_active = np.zeros(self._num_parcels)
# rhos_mean_active.fill(np.nan)
# find active sand
findactivesand = (
self._parcels.dataset.D < _SAND_SIZE
) * self._active_parcel_records # since find active already sets all prior timesteps to False, we can use D for all timesteps here.
vol_act_sand = self._parcels.calc_aggregate_value(
xr.Dataset.sum,
"volume",
at="link",
filter_array=findactivesand,
fill_value=0.0,
)
frac_sand = np.zeros_like(self._vol_act)
frac_sand[self._vol_act != 0.0] = (
vol_act_sand[self._vol_act != 0.0] / self._vol_act[self._vol_act != 0.0]
)
frac_sand[np.isnan(frac_sand)] = 0.0
# Calc attributes for each link, map to parcel arrays
for i in range(self._grid.number_of_links):
active_here = np.nonzero(
np.logical_and(Linkarray == i, Activearray == _ACTIVE)
)[0]
d_act_i = Darray[active_here]
vol_act_i = Volarray[active_here]
rhos_act_i = Rhoarray[active_here]
vol_act_tot_i = np.sum(vol_act_i)
self._d_mean_active[i] = np.sum(d_act_i * vol_act_i) / (vol_act_tot_i)
if vol_act_tot_i > 0:
self._rhos_mean_active[i] = np.sum(rhos_act_i * vol_act_i) / (
vol_act_tot_i
)
else:
self._rhos_mean_active[i] = np.nan
D_mean_activearray[Linkarray == i] = self._d_mean_active[i]
frac_sand_array[Linkarray == i] = frac_sand[i]
vol_act_array[Linkarray == i] = self._vol_act[i]
Sarray[Linkarray == i] = self._grid.at_link["channel_slope"][i]
Harray[Linkarray == i] = self._grid.at_link["flow_depth"][i]
Larray[Linkarray == i] = self._grid.at_link["reach_length"][i]
active_layer_thickness_array[Linkarray == i] = self._active_layer_thickness[
i
]
# Wilcock and Crowe calculate transport for all parcels (active and inactive)
taursg = _calculate_reference_shear_stress(
self._fluid_density, R, self._g, D_mean_activearray, frac_sand_array
)
frac_parcel = np.nan * np.zeros_like(Volarray)
frac_parcel[vol_act_array != 0.0] = (
Volarray[vol_act_array != 0.0] / vol_act_array[vol_act_array != 0.0]
)
b = 0.67 / (1.0 + np.exp(1.5 - Darray / D_mean_activearray))
tau = self._fluid_density * self._g * Harray * Sarray
tau = np.atleast_1d(tau)
taur = taursg * (Darray / D_mean_activearray) ** b
tautaur = tau / taur
tautaur_cplx = tautaur.astype(np.complex128)
# ^ work around needed b/c np fails with non-integer powers of negative numbers
W = 0.002 * np.power(tautaur_cplx.real, 7.5)
W[tautaur >= 1.35] = 14 * np.power(
(1 - (0.894 / np.sqrt(tautaur_cplx.real[tautaur >= 1.35]))), 4.5
)
active_parcel_idx = Activearray == _ACTIVE
# compute parcel virtual velocity, m/s
self._pvelocity[active_parcel_idx] = (
W.real[active_parcel_idx]
* (tau[active_parcel_idx] ** (3.0 / 2.0))
* frac_parcel[active_parcel_idx]
/ (self._fluid_density ** (3.0 / 2.0))
/ self._g
/ R[active_parcel_idx]
/ active_layer_thickness_array[active_parcel_idx]
)
self._pvelocity[np.isnan(self._pvelocity)] = 0.0
if np.max(self._pvelocity) > 1:
warnings.warn(
"NetworkSedimentTransporter: Maximum parcel virtual velocity exceeds 1 m/s"
)
# Assign those things to the grid -- might be useful for plotting
self._grid.at_link["sediment_total_volume"] = self._vol_tot
self._grid.at_link["sediment__active__volume"] = self._vol_act
self._grid.at_link["sediment__active__sand_fraction"] = frac_sand
def pool3d_ncdhw_python(
np_data,
kernel,
strides,
padding,
out_shape,
pool_type,
count_include_pad=True,
ceil_mode=False,
dtype="float32",
):
"""baseline for max_pool3d and avg_pool3d, default layout is "NCDHW"""
# fmt: off
in_n, in_c, in_d, in_h, in_w = in_shape = np_data.shape
if isinstance(kernel, int):
k_d = k_h = k_w = kernel
else:
k_d, k_h, k_w = kernel
if isinstance(strides, int):
s_d = s_h = s_w = strides
else:
s_d, s_h, s_w = strides
if isinstance(padding, int):
pf = pt = pl = pk = pb = pr = padding
else:
pf, pt, pl, pk, pb, pr = padding
if ceil_mode:
assert out_shape[2] == int(
math.ceil(float(in_shape[2] - k_d + pf + pk) / s_d) + 1)
assert out_shape[3] == int(
math.ceil(float(in_shape[3] - k_h + pt + pb) / s_h) + 1)
assert out_shape[4] == int(
math.ceil(float(in_shape[4] - k_w + pl + pr) / s_w) + 1)
else:
assert out_shape[2] == int(
math.floor(float(in_shape[2] - k_d + pf + pk) / s_d) + 1)
assert out_shape[3] == int(
math.floor(float(in_shape[3] - k_h + pt + pb) / s_h) + 1)
assert out_shape[4] == int(
math.floor(float(in_shape[4] - k_w + pl + pr) / s_w) + 1)
fill_value = tvm.tir.const(0.0, dtype).value
if not (count_include_pad) and pool_type == 'max':
fill_value = tvm.te.min_value(dtype).value
pad_np = np.full(shape=(in_n, in_c, in_d + pf + pk, in_h + pt + pb,
in_w + pl + pr),
fill_value=fill_value,
dtype=dtype)
no_zero = (range(in_n), range(in_c), (range(pf, in_d + pf)),
(range(pt, in_h + pt)), (range(pl, in_w + pl)))
pad_np[np.ix_(*no_zero)] = np_data
ret_np = np.zeros(shape=out_shape).astype(dtype)
if pool_type == 'avg':
for k in range(out_shape[2]):
for i in range(out_shape[3]):
for j in range(out_shape[4]):
if count_include_pad:
ret_np[:, :, k, i, j] = \
np.mean(pad_np[:, :, k * s_d: k * s_d + k_d,
i * s_h: i * s_h + k_h,
j * s_w: j * s_w + k_w], axis=(2, 3, 4))
else:
pad_count = np.sum(pad_np[:, :, k * s_d:k * s_d + k_d,
i * s_h:i * s_h + k_h,
j * s_w:j * s_w + k_w] > 0,
axis=(2, 3, 4))
ret_np[:, :, k, i, j] = np.sum(
pad_np[:, :, k * s_d:k * s_d + k_d, i *
s_h:i * s_h + k_h, j * s_w:j * s_w + k_w],
axis=(2, 3, 4)) / np.maximum(pad_count, 1)
elif pool_type == 'max':
for k in range(out_shape[2]):
for i in range(out_shape[3]):
for j in range(out_shape[4]):
ret_np[:, :, k, i,
j] = np.max(pad_np[:, :, k * s_d:k * s_d + k_d,
i * s_h:i * s_h + k_h,
j * s_w:j * s_w + k_w],
axis=(2, 3, 4))
else:
raise ValueError("pool type {} is not supported".format(pool_type))
ret_np = np.maximum(ret_np, fill_value)
# fmt: on
return ret_np
def transform(self, X: dt.Frame, **kwargs):
"""
Uses fitted models (1 per time group) to predict the target
:param X: Datatable Frame containing the features
:return: FB Prophet predictions
"""
# Get the logger if it exists
logger = self.get_experiment_logger()
tmp_folder = self._create_tmp_folder(logger)
n_jobs = self._get_n_jobs(logger, **kwargs)
# Reduce X to TGC
tgc_wo_time = list(np.setdiff1d(self.tgc, self.time_column))
# Change date feature name to match Prophet requirements
X = self.convert_to_prophet(X)
y_predictions = self.predict_with_average_model(X, tgc_wo_time)
y_predictions.columns = ['average_pred']
# Go through groups
for grp_col in tgc_wo_time:
# Get the unique dates to be predicted
X_groups = X[['ds', grp_col]].groupby(grp_col)
# Go though the groups and predict only top
XX_paths = []
model_paths = []
def processor(out, res):
out.append(res)
num_tasks = len(X_groups)
pool_to_use = small_job_pool
pool = pool_to_use(logger=None,
processor=processor,
num_tasks=num_tasks,
max_workers=n_jobs)
for _i_g, (key, X_grp) in enumerate(X_groups):
# Just log where we are in the fitting process
if (_i_g + 1) % max(1, num_tasks // 20) == 0:
loggerinfo(
logger, "FB Prophet : %d%% of groups predicted" %
(100 * (_i_g + 1) // num_tasks))
# Create dict key to store the min max scaler
grp_hash = self.get_hash(key)
X_path = os.path.join(tmp_folder,
"fbprophet_Xt" + str(uuid.uuid4()))
if grp_hash not in self.grp_models[grp_col]:
# unseen groups
XX = pd.DataFrame(np.full((X_grp.shape[0], 1), np.nan),
columns=['yhat'])
XX.index = X_grp.index
save_obj(XX, X_path)
XX_paths.append(X_path)
continue
if self.grp_models[grp_col][grp_hash] is None:
# known groups but not enough train data
XX = pd.DataFrame(np.full((X_grp.shape[0], 1), np.nan),
columns=['yhat'])
XX.index = X_grp.index
save_obj(XX, X_path)
XX_paths.append(X_path)
continue
model = self.grp_models[grp_col][grp_hash]
model_path = os.path.join(
tmp_folder, "fbprophet_modelt" + str(uuid.uuid4()))
save_obj(model, model_path)
save_obj(X_grp, X_path)
model_paths.append(model_path)
args = (model_path, X_path, self.priors[grp_col][grp_hash],
tmp_folder)
kwargs = {}
pool.submit_tryget(
None,
MyProphetOnSingleGroupsTransformer_transform_async,
args=args,
kwargs=kwargs,
out=XX_paths)
pool.finish()
y_predictions[f'{grp_col}_pred'] = pd.concat(
(load_obj(XX_path) for XX_path in XX_paths),
axis=0).sort_index()
for p in XX_paths + model_paths:
remove(p)
# Now we can invert scale
# But first get rid of NaNs
for grp_col in tgc_wo_time:
# Add time group to the predictions, will be used to invert scaling
y_predictions[grp_col] = X[grp_col]
# Fill NaN
y_predictions[f'{grp_col}_pred'] = y_predictions[
f'{grp_col}_pred'].fillna(y_predictions['average_pred'])
# Go through groups and recover the scaled target for knowed groups
if len(tgc_wo_time) > 0:
X_groups = y_predictions.groupby(tgc_wo_time)
else:
X_groups = [([None], y_predictions)]
for _f in [f'{grp_col}_pred'
for grp_col in tgc_wo_time] + ['average_pred']:
inverted_ys = []
for key, X_grp in X_groups:
grp_hash = self.get_hash(key)
# Scale target for current group
if grp_hash in self.scalers.keys():
inverted_y = self.scalers[grp_hash].inverse_transform(
X_grp[[_f]])
else:
inverted_y = self.general_scaler.inverse_transform(
X_grp[[_f]])
# Put back in a DataFrame to keep track of original index
inverted_df = pd.DataFrame(inverted_y, columns=[_f])
inverted_df.index = X_grp.index
inverted_ys.append(inverted_df)
y_predictions[_f] = pd.concat(tuple(inverted_ys),
axis=0).sort_index()[_f]
self._clean_tmp_folder(logger, tmp_folder)
y_predictions.drop(tgc_wo_time, axis=1, inplace=True)
self._output_feature_names = [
f'{self.display_name}{orig_feat_prefix}{self.time_column}{extra_prefix}{_f}'
for _f in y_predictions
]
self._feature_desc = self._output_feature_names
return y_predictions
a = np.array([1, 2, 3, 4, 5]) # Create a rank 1 array print(type(a)) # Prints "<class 'numpy.ndarray'>" print(a.shape) # Prints "(3,)" print(a[0], a[1], a[2]) # Prints "1 2 3" a[0] = 5 # Change an element of the array print(a) # Prints "[5, 2, 3]" b = np.array([[1, 2, 3], [4, 5, 6]]) # Create a rank 2 array print(b.shape) # Prints "(2, 3)" print(b[0, 0], b[0, 1], b[1, 0]) # Prints "1 2 4" a = np.zeros((2, 2)) # Create an array of all zeros print(a) # Prints "[[ 0. 0.] # [ 0. 0.]]" b = np.ones((1, 2)) # Create an array of all ones print(b) # Prints "[[ 1. 1.]]" c = np.full((2, 2), 7) # Create a constant array print(c) # Prints "[[ 7. 7.] # [ 7. 7.]]" d = np.eye(2) # Create a 2x2 identity matrix print(d) # Prints "[[ 1. 0.] # [ 0. 1.]]" e = np.random.random((2, 2)) # Create an array filled with random values print(e) # Might print "[[ 0.91940167 0.08143941] # [ 0.68744134 0.87236687]]"
if count1 >= count2 and count1 >= count3:
prediction = 1
if count2 >= count1 and count2 >= count3:
prediction = 2
if count3 >= count1 and count3 >= count2:
prediction = 3 # 取出现次数最多的为预测值
if prediction == test[0, 4]:
accuracy += 1
Accuracy = accuracy / 150
print("Iris数据集的最近邻准确率为：", Accuracy)
return Accuracy
x = iris.iloc[:, 0:4]
x = np.mat(x)
a = np.full((50, 1), 1)
b = np.full((50, 1), 2)
c = np.full((50, 1), 3)
Res = np.zeros(50)
d = np.append(a, b, axis=0)
d = np.append(d, c, axis=0)
X = np.append(x, d, axis=1) # 将数据集中的标签更换为1，2，3
for m in range(10):
k = m + 1
Res[m] = k_nn(X)
'''
# 绘制 k与分类准确率的图像
import matplotlib.pyplot as plt
