def main():
""" Run the script as a stand alone application. """
start_year = 1929
stop_year = 1929
print("\n")
# ---- Individual Functions
start = time.time()
temperatures = [[year,
global_mean_temp(year),
global_min_temp(year),
global_max_temp(year)]
for year in xrange(start_year, stop_year + 1)]
for year, mean, min, max in temperatures:
print("{}\t{:.1f}\t{:.1f}\t{:.1f}\t{}"\
.format(year, mean[0], min, max, mean[1]))
print("\nUsing individual functions took {} seconds."\
.format(round(time.time() - start, 2)))
print("\n")
# ---- Single Function
start = time.time()
temperatures = [global_summary(year)
for year in range(start_year, stop_year + 1)]
for year, mean, min, max, num_obs in temperatures:
print("{}\t{:.1f}\t{:.1f}\t{:.1f}\t{}"\
.format(year, mean, min, max, num_obs))
print("\nUsing a combined function took {} seconds."\
.format(round(time.time() - start, 2)))
def GetScalingRatios(self, resolution=None, places=None):
"""
Get the scaling ratios required to upsample an image to `resolution`.
If resolution is None, then assume it will be upsampled to the native
destination resolution. See Dataset.GetNativeResolution()
If places is not None, rounds the ratios to the number of decimal
places specified.
"""
if resolution is None:
resolution = self.GetNativeResolution(transform=None)
# Get the pixel dimensions in map units. There is no custom transform,
# because it makes no sense to compute a pixel ratio for a
# reprojection.
spatial_ref = self.GetSpatialReference()
dst_pixel_width, dst_pixel_height = spatial_ref.GetPixelDimensions(
resolution=resolution
)
src_pixel_width, src_pixel_height = self.GetPixelDimensions()
xscale = abs(src_pixel_width / dst_pixel_width)
yscale = abs(src_pixel_height / dst_pixel_height)
if places is not None:
xscale = round(xscale, places)
yscale = round(yscale, places)
return XY(x=xscale, y=yscale)
def fprop_roipooling_ref(fm, rois, fm_channel, fm_height, fm_width, bsz, rois_per_image, H, W):
feature_maps = fm.reshape(fm_channel, fm_height, fm_width, bsz)
rois_per_batch = rois_per_image * bsz
outputs = np.zeros((fm_channel, H, W, rois_per_batch))
# combine the feature map with ROIs
for b_id in range(rois_per_batch):
[idx, xmin, ymin, xmax, ymax] = rois[b_id]
xmin = int(round(xmin * spatial_scale))
xmax = int(round(xmax * spatial_scale))
ymin = int(round(ymin * spatial_scale))
ymax = int(round(ymax * spatial_scale))
roi_width = max(xmax - xmin + 1, 1)
roi_height = max(ymax - ymin + 1, 1)
stride_h = float(roi_height) / H
stride_w = float(roi_width) / W
for h_out in range(H):
sliceh, _ = _fprop_slice_np(h_out, stride_h, fm_height, ymin)
if sliceh.stop <= sliceh.start:
continue
for w_out in range(W):
slicew, _ = _fprop_slice_np(w_out, stride_w, fm_width, xmin)
if slicew.stop <= slicew.start:
continue
else:
array_I = feature_maps[:, sliceh, slicew, int(idx)].reshape(
fm_channel, -1)
outputs[:, h_out, w_out, b_id] = np.max(array_I, axis=1)
return outputs.reshape(-1, rois_per_batch)
def GetWorldScalingRatios(self, resolution=None, places=None):
"""
Get the scaling ratios required to upsample for the whole world.
If resolution is None, then assume it will be upsampled to the native
destination resolution. See Dataset.GetNativeResolution()
If places is not None, rounds the ratios to the number of decimal
places specified.
"""
if resolution is None:
resolution = self.GetNativeResolution()
spatial_ref = self.GetSpatialReference()
world = spatial_ref.GetWorldExtents().dimensions
src_pixel_sizes = XY(x=world.x / self.RasterXSize,
y=world.y / self.RasterYSize)
dst_pixel_sizes = spatial_ref.GetPixelDimensions(resolution=resolution)
xscale = abs(src_pixel_sizes.x / dst_pixel_sizes.x)
# Make sure that yscale fits within the whole world
yscale = min(xscale, abs(src_pixel_sizes.y / dst_pixel_sizes.y))
if places is not None:
xscale = round(xscale, places)
yscale = round(yscale, places)
return XY(x=xscale, y=yscale)
def test_pearson_one_target_one_feature(self, idadf):
data = idadf._table_def()
columns = list(data.loc[data['VALTYPE'] == "NUMERIC"].index)
if len(columns) > 1:
result = pearson(idadf, target = columns[0], features=[columns[1]])
assert(isinstance(result, float))
result2 = pearson(idadf, target = columns[1], features=[columns[0]])
assert(round(result,3) == round(result2,3)) # symmetry
def round(f):
# P3's builtin round differs from P2 in the following manner:
# * it rounds half to even rather than up (away from 0)
# * round(-0.) loses the sign (it returns -0 rather than 0)
# * round(x) returns an int rather than a float
#
# this compatibility shim implements Python 2's round in terms of
# Python 3's so that important rounding error under P3 can be
# trivially fixed, assuming the P2 behaviour to be debugged and
# correct.
roundf = builtins.round(f)
if builtins.round(f + 1) - roundf != 1:
return f + math.copysign(0.5, f)
# copysign ensures round(-0.) -> -0 *and* result is a float
return math.copysign(roundf, f)
def hhmmss(t):
"""
:param t: time elapsed (in seconds)
Returns the string representation of ``t`` in format: ``hhmmss``
"""
return str(datetime.timedelta(seconds=round(t)))
def test_px_to_em_change_base(self):
base = 16
for pixels in range(-1000, 1001):
expected = round(pixels / base, 4)
expected = str(expected) + 'em'
actual = settings.px_to_em(pixels=pixels)
self.assertEqual(actual, expected, msg=str(actual) + ' vs ' + str(expected))
def test_px_to_em_int_input(self):
base = 16
for pixels in range(-1000, 1001):
expected = round(pixels / base, 4)
expected = str(expected) + 'em'
actual = settings.px_to_em(pixels=pixels)
self.assertEqual(actual, str(expected), msg=pixels)
def test_px_to_em_typecast_string_input(self):
base = 16
for pixels in range(-1000, 1001):
expected = round(pixels / base, 4)
expected = str(expected) + 'em'
actual = unittest_settings.px_to_em(pixels=str(pixels)) # typecast to string str()
self.assertEqual(actual, str(expected), msg=pixels)
def probe(self):
"""
Gets the connected probe value for this channel
:type: `float`
"""
return round(1 / float(self._parent.query("CH{}:PRO?".format(self._idx))), 0)
def bprop_roipooling_ref(fm, rois, error, fm_channel, fm_height, fm_width,
bsz, rois_per_image, H, W):
"""
Function to perform a bprop of ROIPooling. It uses a different way from the
that in CPU backend
"""
feature_maps = fm.reshape(fm_channel, fm_height, fm_width, bsz)
rois_per_batch = rois_per_image * bsz
error_in = error.reshape(fm_channel, H, W, rois_per_batch)
delta = np.zeros(feature_maps.shape).reshape(fm_channel, fm_height, fm_width, bsz)
# combine the feature map with ROIs
for b_id in range(rois_per_batch):
[idx, xmin, ymin, xmax, ymax] = rois[b_id]
xmin = int(round(xmin * spatial_scale))
xmax = int(round(xmax * spatial_scale))
ymin = int(round(ymin * spatial_scale))
ymax = int(round(ymax * spatial_scale))
roi_width = max(xmax - xmin + 1, 1)
roi_height = max(ymax - ymin + 1, 1)
stride_h = float(roi_height) / float(H)
stride_w = float(roi_width) / float(W)
for h_out in range(H):
sliceh, lenh = _fprop_slice_np(h_out, stride_h, fm_height, ymin)
if sliceh.stop <= sliceh.start:
continue
for w_out in range(W):
slicew, lenw = _fprop_slice_np(w_out, stride_w, fm_width, xmin)
if slicew.stop <= slicew.start:
continue
else:
array_I = feature_maps[:, sliceh, slicew, int(idx)].reshape(
fm_channel, -1)
max_idx = np.argmax(array_I, axis=1)
delta_view = delta[:, sliceh, slicew, int(idx)].reshape(
fm_channel, -1)
delta_view[
list(range(fm_channel)), max_idx] += error_in[:, h_out, w_out, b_id]
delta[:, sliceh, slicew, int(idx)] = delta_view.reshape(fm_channel,
lenh,
lenw)
return delta
def test_px_to_em_float_input(self):
base = 16
# Thank you: http://stackoverflow.com/questions/477486/python-decimal-range-step-value#answer-477506
for pixels in range(-11, 11, 1):
pixels /= 10.0
expected = round(float(pixels) / float(base), 4)
expected = str(expected) + 'em'
actual = settings.px_to_em(pixels=pixels)
self.assertEqual(actual, str(expected), msg=str(pixels) + ': ' + str(actual) + ' vs ' + str(expected))
def bisekt(self, lowerBound, higherBound):
arr = []
middleBound = ((higherBound - lowerBound) / 2) + lowerBound
arr.append (self.afterAyear(higherBound))
arr.append (self.afterAyear(middleBound))
arr.append(self.afterAyear(lowerBound))
index = bisect(arr, 0)
# exit clause for recursive function
if (math.ceil(middleBound * 100) / 100) == higherBound:
return[higherBound, arr[0]]
else:
# find out whether balance=0 after 12 months is between lowerBound and middleBound or higherBound and middleBound
if index == 1:
lowerBound = middleBound
elif index == 2:
higherBound = middleBound
return self.bisekt(round(lowerBound, 2), round(higherBound, 2))
def deepdream(image, iter_n=10, octave_n=4, octave_scale=1.4, name="Deep Dream"):
model = DreamModel(model_path=args.data_dir)
detail = None
scales = [octave_scale ** -o for o in reversed(list(range(octave_n)))]
for o_idx, scale in enumerate(scales):
octave_shape = (3, round(image.shape[1] * scale), round(image.shape[2] * scale))
octave_base = zoom_to(image.as_tensor(), octave_shape)
detail = np.zeros_like(octave_base) if detail is None else zoom_to(detail, octave_shape)
dream = DeepImage(octave_base + detail)
model.initialize(dream)
for i in range(iter_n):
dream.take_step(model)
ofile = get_numbered_file(args.dream_file, o_idx * iter_n + i)
dream.save_image(ofile)
detail = dream.as_tensor() - octave_base
return dream
def print_css_stats(file_name=''):
# ``file_name`` the full file_name excluding extension e.g. 'blowdry' or 'site'.
# Assumes that the extensions to append to the file_name are '.css' and '.min.css'.
# Print the size of a file_name.
css_file = file_name + '.css'
min_file = file_name + '.min.css'
css_dir = path.join(settings.css_directory, css_file) # Get full file path.
min_dir = path.join(settings.css_directory, min_file)
css_size = stat(css_dir).st_size # Get file size in bytes.
min_size = stat(min_dir).st_size
percent_reduced = round(float(min_size) / float(css_size) * float(100), 1) # Calculate percentage size reduced.
css_kb = round(float(css_size) / float(1000), 1) # Convert to kilobytes.
min_kb = round(float(min_size) / float(1000), 1)
print('\n' + css_file + ':\t', css_kb, 'kb')
print(min_file + ':', min_kb, 'kb')
print('CSS file size reduced by', str(percent_reduced) + '%.')
def serialized(array, nsig, ndec):
if array.ndim > 0:
return '[{}]'.format(','.join(serialized(a, nsig, ndec) for a in array))
if not numpy.isfinite(array): # nan, inf
return str(array)
a = builtins.round(float(array) * 10**ndec)
if a == 0:
return '0'
while abs(a) >= 10**nsig:
a //= 10
ndec -= 1
return '{}e{}'.format(a, -ndec)
def round_to(self, base):
r"""Round to a specific base (like it's required for a grid)
:param base: base we want to round to
:return: rounded point
>>> from KicadModTree import *
>>> Vector2D(0.1234, 0.5678).round_to(0.01)
"""
if base == 0 or base is None:
return self.__copy__()
return Vector2D([round(v / base) * base for v in self])
def print_css_stats(file_name=''):
""" ``file_name`` the full file_name excluding extension e.g. 'blowdry' or 'site'.
Assumes that the extensions to append to the file_name are '.css' and '.min.css'.
Print the size of a file_name.
:type file_name: str
:param file_name: Name of the CSS files.
:return: None
"""
css_file = file_name + '.css'
min_file = file_name + '.min.css'
css_dir = path.join(settings.css_directory, css_file) # Get full file path.
min_dir = path.join(settings.css_directory, min_file)
css_size = stat(css_dir).st_size # Get file size in Bytes.
min_size = stat(min_dir).st_size
try:
percent_reduced = round( # Calculate percentage size reduced.
float(100) - float(min_size) / float(css_size) * float(100),
1 # Precision
)
except ZeroDivisionError:
percent_reduced = round(0.0, 1)
css_kb = round(float(css_size) / float(1000), 1) # Convert to kiloBytes.
min_kb = round(float(min_size) / float(1000), 1)
css_stats = (
'\n' + str(css_file) + ':\t ' + str(css_kb) + 'kB\n' +
str(min_file) + ': ' + str(min_kb) + 'kB\n' +
'CSS file size reduced by ' + str(percent_reduced) + '%.'
)
logging.debug(css_stats)
print(css_stats)
def GetTmsExtents(self, resolution=None, transform=None):
"""
Returns (lower-left, upper-right) TMS tile coordinates.
The upper-right coordinates are excluded from the range, while the
lower-left are included.
"""
if resolution is None:
resolution = self.GetNativeResolution(transform=transform)
# Get the tile dimensions in map units
if transform is None:
spatial_ref = self.GetSpatialReference()
else:
spatial_ref = transform.dst_ref
tile_width, tile_height = spatial_ref.GetTileDimensions(
resolution=resolution
)
# Validate that the native resolution extents are tile-aligned.
extents = self.GetTiledExtents(transform=transform)
pixel_sizes = spatial_ref.GetPixelDimensions(resolution=resolution)
if not extents.almost_equal(self.GetExtents(transform=transform),
delta=min(*pixel_sizes)):
raise UnalignedInputError('Dataset is not aligned to TMS grid')
# Correct for origin, because you can't do modular arithmetic on
# half-tiles.
left, bottom = spatial_ref.OffsetPoint(*extents.lower_left)
right, top = spatial_ref.OffsetPoint(*extents.upper_right)
# Divide by number of tiles
return Extents(lower_left=XY(int(round(left / tile_width)),
int(round(bottom / tile_height))),
upper_right=XY(int(round(right / tile_width)),
int(round(top / tile_height))))
def _get_split_key(keys, num_splits):
"""Given a list of keys and a number of splits find the keys to split on.
Args:
keys: the list of keys.
num_splits: the number of splits.
Returns:
A list of keys to split on.
"""
# If the number of keys is less than the number of splits, we are limited
# in the number of splits we can make.
if not keys or (len(keys) < (num_splits - 1)):
return keys
# Calculate the number of keys per split. This should be KEYS_PER_SPLIT,
# but may be less if there are not KEYS_PER_SPLIT * (numSplits - 1) scatter
# entities.
#
# Consider the following dataset, where - represents an entity and
# * represents an entity that is returned as a scatter entity:
# ||---*-----*----*-----*-----*------*----*----||
# If we want 4 splits in this data, the optimal split would look like:
# ||---*-----*----*-----*-----*------*----*----||
# | | |
# The scatter keys in the last region are not useful to us, so we never
# request them:
# ||---*-----*----*-----*-----*------*---------||
# | | |
# With 6 scatter keys we want to set scatter points at indexes: 1, 3, 5.
#
# We keep this as a float so that any "fractional" keys per split get
# distributed throughout the splits and don't make the last split
# significantly larger than the rest.
num_keys_per_split = max(1.0, float(len(keys)) / (num_splits - 1))
split_keys = []
# Grab the last sample for each split, otherwise the first split will be too
# small.
for i in range(1, num_splits):
split_index = int(round(i * num_keys_per_split) - 1)
split_keys.append(keys[split_index])
return split_keys
def get_estimated_num_splits(project, namespace, query, datastore):
"""Computes the number of splits to be performed on the given query."""
try:
estimated_size_bytes = ReadFromDatastore.get_estimated_size_bytes(
project, namespace, query, datastore)
logging.info('Estimated size bytes for query: %s', estimated_size_bytes)
num_splits = int(min(ReadFromDatastore._NUM_QUERY_SPLITS_MAX, round(
(float(estimated_size_bytes) /
ReadFromDatastore._DEFAULT_BUNDLE_SIZE_BYTES))))
except Exception as e:
logging.warning('Failed to fetch estimated size bytes: %s', e)
# Fallback in case estimated size is unavailable.
num_splits = ReadFromDatastore._NUM_QUERY_SPLITS_MIN
return max(num_splits, ReadFromDatastore._NUM_QUERY_SPLITS_MIN)
def Round(a,base_place=0,base=10):
'''
implement Rounding to bases other than 10.
known deficiencies:
fails with decimal input
fails with fractions input
Beware of 0 < base < 1 since b**(-p) == (1/b)**(p)
>>> [Round(x,-1,5) for x in (142,143,147,)]
[140, 145, 145]
>>>
>>> [(p,Round(12.345,p,2)) for p in range(-3,3)]
[(-3, 16.0), (-2, 12.0), (-1, 12.0), (0, 12.0), (1, 12.5), (2, 12.25)]
>>>
>>> # Use for base < 1
>>> # one might want decimal input
>>> # Round to the nearest US nickel
>>> no_pennies = Round(12.07,-1,0.05)
>>> print('{0:.2f}'.format(no_pennies))
12.05
>>> # The advertising game
>>> price_per_gallon = Round(12.379999999999,-1,1/100)
>>> print('{0:.2f}'.format(price_per_gallon))
12.38
>>>
>>> # round to nearest multiple of square root of two
>>> x = Round(12.379999999999,1/2,2)
>>> print (x / 2**(1/2))
9.0
>>>
'''
# consider using sign transfer for negative a
if base == 10:
return builtins.round(a,base_place)
if base <= 0:
raise ValueError('base too complex')
b = base**base_place
return type(a)(int(a*b+0.5)/b)
def test_get_bounding_square(self):
s1 = self.proj.get_bounding_square()
s2 = self.proj.get_bounding_square(5)
# is a square
self.assertAlmostEqual(s1[1] - s1[0], s1[3] - s1[2])
self.assertAlmostEqual(s2[1] - s2[0], s2[3] - s2[2])
# s2 around s1
self.assertTrue(s2[0] < s1[0] and s1[0] < s2[1])
self.assertTrue(s2[1] > s1[1] and s1[1] > s2[0])
self.assertTrue(s2[2] < s1[2] and s1[2] < s2[3])
self.assertTrue(s2[3] > s1[3] and s1[3] > s2[2])
# All points inside s1
for point in self.proj.proj_graph.nodes():
self.assertLessEqual(round(s1[0], 7), round(point[0], 7), msg="Point {} outside of "
"bounding square {} at the LEFT".format(point, s1))
self.assertLessEqual(round(point[0], 7), round(s1[1], 7), msg="Point {} outside of "
"bounding square {} at the RIGHT".format(point, s1))
self.assertLessEqual(round(s1[2], 7), round(point[1], 7), msg="Point {} outside of "
"bounding square {} at the BOTTOM".format(point, s1))
self.assertLessEqual(round(point[1], 7), round(s1[3], 7), msg="Point {} outside of "
"bounding square {} at the TOP".format(point, s1))
def px_to_em(pixels):
""" Convert a numeric value from px to em using ``settings.base`` as the unit conversion factor.
**Rules:**
- ``pixels`` shall only contain [0-9.-].
- Inputs that contain any other value are simply passed through unchanged.
- Default ``base`` is 16 meaning ``16px = 1rem``
**Note:** Does not check the ``property_name`` or ``use_em`` values. Rather, it blindly converts
whatever input is provided. The calling method is expected to know what it is doing.
Rounds float to a maximum of 4 decimal places.
:type pixels: str, int, float
:param pixels: A numeric value with the units stripped.
:return: (str)
- If the input is convertible return the converted number as a string with the units ``em``
appended to the end.
- If the input is not convertible return the unprocessed input.
>>> from blowdrycss_settings import px_to_em
>>> # settings.use_em = True
>>> px_to_em(pixels='-16.0')
-1em
>>> # settings.use_em = False
>>> px_to_em(pixels='42px')
42px
>>> # Invalid input passes through.
>>> px_to_em(pixels='invalid')
invalid
"""
if set(str(pixels)) <= set(digits + '-.'):
em = float(pixels) / float(base)
em = round(em, 4)
em = str(em) + 'em' # Add 'em'.
return em
return pixels
def _rounded_compare(val1, val2):
print('Comparing {} and {} using round()'.format(val1, val2))
return builtins.round(val1) == builtins.round(val2)
def round_to(x, step):
return step * round(old_div(x, step))
from builtins import round
import matplotlib.pyplot as plt
# Assumptions: ScreenSize = [34.5,19.5] cm, eyeLocation = [0,11.75,60] cm, DistOfCamFromTheScreen = -2 cm
ScreenSize = [34.5, 19.5] # cm units (Alon's Computer)
DistOfCamFromTheScreen = 2.0 # cm units
eyeLocation = [0.0, ScreenSize[1] / 2 + DistOfCamFromTheScreen, 60.0] # cm units (relative to the camera)
histogramX = []
histogramY = []
# path joining version for other paths
DIR = os.path.join(os.getcwd(), r'UnityEyes/imgs/')
DataSetSize = len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))])
for i in range(1, round(DataSetSize / 2)):
with open(r'UnityEyes/imgs/{0}.json'.format(i)) as jsonFile:
data = json.load(jsonFile)
eyeDetails = data['eye_details']
Vec = eyeDetails['look_vec']
lookVec = Vec[1:-1].split(',')
T = eyeLocation[2] / float(lookVec[2])
Xscreen = eyeLocation[0] + T * float(lookVec[0]) + ScreenSize[0] / 2 # The sign is + because the camera flips the look vector in x axis
Yscreen = eyeLocation[1] - T * float(lookVec[1]) - DistOfCamFromTheScreen
histogramX.append(Xscreen)
histogramY.append(Yscreen)
bins=100
plt.hexbin(histogramX, histogramY, bins=bins, gridsize=1000) # <- You need to do the hexbin plot
def _parameter_value_to_ring_value(self, value, minv, maxv):
vrange = self._ring_value_range
index = int(
round(
(value - minv) * old_div(len(vrange) - 1, float(maxv - minv))))
return vrange[index]
def round(x):
"""Rounds the given number to a globally constant precision."""
return builtins.round(x, this._precision_digits)
def _roundFigures(x, n):
return round(x, -int(np.floor(np.log10(abs(x)))) + (n - 1))
def to_user_range_quantized(minv, maxv):
user_range_transform = to_user_range(minv, maxv)
return lambda v, s: int(round(user_range_transform(v, s)))
def main(args):
files = args.cgfiles
#Uncomment the following line to display the files in a random order.
#random.shuffle(files)
#Prepare the pyplot figure
totalFigures = len(files)
figuresPerLine = int(math.ceil(math.sqrt(totalFigures)))
fig, ax = plt.subplots(int(math.ceil(totalFigures / figuresPerLine)),
figuresPerLine,
squeeze=False,
figsize=(8, 8))
#Background color of figure (not plot)
if args.style == "WOB":
fig.patch.set_facecolor('black')
#Plot one projection per file.
for i, file_ in enumerate(files):
#get the subplot axes (Note: axes != axis in matplotlib)
current_axes = ax[i // figuresPerLine, i % figuresPerLine]
#Parse the file
cg = ftmc.CoarseGrainRNA(file_)
# Random projection direction, if no direction present in the file
if args.proj_direction:
direction = list(map(float, args.proj_direction.split(",")))
elif cg.project_from is not None:
direction = cg.project_from
else:
direction = ftuv.get_random_vector()
#Generate the projection object
proj = ftmp.Projection2D(cg,
direction,
rotation=180,
project_virtual_atoms=args.virtual_atoms)
#Simulate a reduced resolution of the image.
if args.condense:
proj.condense(args.condense)
target_elems = []
if args.show_distances:
try:
num_elems = int(args.show_distances)
except:
target_elems = args.show_distances.split(",")
else:
if num_elems > len(proj._coords.keys()):
raise ValueError(
"--show-distances must not be greater {} for the current projection ({}:'{}')"
.format(len(proj._coords.keys()), i, file_))
elems = list(proj._coords.keys())
random.shuffle(elems)
while len(target_elems) < num_elems:
r = random.random()
if r < 0.4:
hairpins = [
x for x in elems
if x[0] == "h" and x not in target_elems
]
if hairpins:
target_elems.append(hairpins[0])
continue
if r < 0.6:
multiloops = [
x for x in elems
if x[0] == "m" and x not in target_elems
]
if multiloops:
target_elems.append(multiloops[0])
continue
others = [x for x in elems if x not in target_elems]
target_elems.append(others[0])
comb = list(it.combinations(target_elems, 2))
#print(comb, target_elems)
if args.label_elements:
target_elems = list(proj._coords.keys())
line2dproperties = {}
if args.style == "BOW":
line2dproperties["color"] = "black"
elif args.style == "WOB":
line2dproperties["color"] = "white"
#Plot the projection #
proj.plot(current_axes,
margin=15,
linewidth=3,
add_labels=set(target_elems),
line2dproperties=line2dproperties,
show_distances=comb,
print_distances=args.print_distances)
#Uncomment to set a substring of the filename as a title
#current_axes.set_title(file[-15:])
#Hide the x- and y axis.
current_axes.get_xaxis().set_visible(False)
current_axes.get_yaxis().set_visible(False)
#Print the projection direction and the filename in the plot.
if args.show_direction or args.p:
current_axes.text(0.01,
0.01,
"Projection direction: ({},{},{})".format(
round(direction[0],
3), round(direction[1], 3),
round(direction[2], 3)),
transform=current_axes.transAxes)
if args.show_filename or args.p:
current_axes.text(
0.01,
0.99,
"File: {}".format(file_),
transform=current_axes.transAxes,
verticalalignment='top',
)
#Change the backgroundcolor of the plot area.
if args.style == "WOB":
current_axes.set_axis_bgcolor('black')
#Hide additional subplots with no projection on them.
for i in range(
len(files),
int(math.ceil(totalFigures / figuresPerLine)) * figuresPerLine):
ax[i // figuresPerLine, i % figuresPerLine].axis('off')
# Reduce the space outside of the plots and between the subplots.
plt.subplots_adjust(left=0.025,
right=0.975,
bottom=0.025,
top=0.975,
wspace=0.05,
hspace=0.05)
if args.out:
for ofname in args.out:
if args.out_path:
ofname = os.path.join(args.out_path, ofname)
ofname = os.path.expanduser(ofname)
plt.savefig(ofname, format=ofname[-3:])
if not args.out or args.show:
#Show the plot and clear it from the internal memory of matplotlib.
plt.show()
def test_multiply_integer_reverse_current_value(self, pi_point):
"""Test adding a PIPoint to an integer via the current value."""
point2 = 1 * pi_point.point
assert round(point2.current_value - (pi_point.values[-1] * 1),
ndigits=7) == 0
for v in sess.graph.get_collection('trainable_variables'):
num_params += np.prod(v.get_shape()).value
print('epoch', 'val loss', 'duration', sep='\t')
run_start = start = timeit.default_timer()
validation_loss = 0
for i in range(len(val_images)//minibatch_size):
minibatch_validation_loss = sess.run(total_loss, feed_dict={
seq_in: val_captions_in [i*minibatch_size:(i+1)*minibatch_size],
seq_len: val_captions_len[i*minibatch_size:(i+1)*minibatch_size],
seq_target: val_captions_out[i*minibatch_size:(i+1)*minibatch_size],
image: val_images[i*minibatch_size:(i+1)*minibatch_size]
})
validation_loss += minibatch_validation_loss
print(0, round(validation_loss, 3), round(timeit.default_timer() - start), sep='\t')
last_validation_loss = validation_loss
trainingset_indexes = list(range(len(train_images)))
for epoch in range(1, max_epochs+1):
random.shuffle(trainingset_indexes)
start = timeit.default_timer()
for i in range(len(trainingset_indexes)//minibatch_size):
minibatch_indexes = trainingset_indexes[i*minibatch_size:(i+1)*minibatch_size]
sess.run(train_step, feed_dict={
seq_in: train_captions_in [minibatch_indexes],
seq_len: train_captions_len[minibatch_indexes],
seq_target: train_captions_out[minibatch_indexes],
image: train_images[minibatch_indexes]
})
def divConn(self, preCellsTags, postCellsTags, connParam):
from .. import sim
''' Generates connections between all pre and post-syn cells based on probability values'''
if sim.cfg.verbose:
print('Generating set of divergent connections (rule: %s) ...' %
(connParam['label']))
# get list of params that have a lambda function
paramsStrFunc = [
param for param in [p + 'Func' for p in self.connStringFuncParams]
if param in connParam
]
# copy the vars into args immediately and work out which keys are associated with lambda functions only once per method
funcKeys = {}
for paramStrFunc in paramsStrFunc:
connParam[paramStrFunc + 'Args'] = connParam[paramStrFunc +
'Vars'].copy()
funcKeys[paramStrFunc] = [
key for key in connParam[paramStrFunc + 'Vars']
if callable(connParam[paramStrFunc + 'Vars'][key])
]
# converted to list only once
postCellsTagsKeys = sorted(postCellsTags)
# calculate hash for post cell gids
hashPostCells = sim.hashList(postCellsTagsKeys)
for preCellGid, preCellTags in preCellsTags.items(
): # for each presyn cell
divergence = connParam['divergenceFunc'][
preCellGid] if 'divergenceFunc' in connParam else connParam[
'divergence'] # num of presyn conns / postsyn cell
divergence = max(min(int(round(divergence)),
len(postCellsTags) - 1), 0)
self.rand.Random123(hashPostCells, preCellGid,
sim.cfg.seeds['conn']) # init randomizer
randSample = self.randUniqueInt(self.rand, divergence + 1, 0,
len(postCellsTags) - 1)
# note: randSample[divergence] is an extra value used only if one of the random postGids coincided with the preGid
postCellsSample = {
postCellsTagsKeys[randSample[divergence]]
if postCellsTagsKeys[i] == preCellGid else postCellsTagsKeys[i]: 0
for i in randSample[0:divergence]
} # dict of selected gids of postsyn cells with removed pre gid
for postCellGid in [c for c in postCellsSample if c in self.gid2lid]:
postCellTags = postCellsTags[postCellGid]
for paramStrFunc in paramsStrFunc: # call lambda functions to get weight func args
# update the relevant FuncArgs dict where lambda functions are known to exist in the corresponding FuncVars dict
for funcKey in funcKeys[paramStrFunc]:
connParam[paramStrFunc +
'Args'][funcKey] = connParam[paramStrFunc +
'Vars'][funcKey](
preCellTags,
postCellTags)
if preCellGid != postCellGid: # if not self-connection
self._addCellConn(connParam, preCellGid,
postCellGid) # add connection
from math import fabs
from builtins import round
# Absolute value
data = -5
data2 = -4.5
data3 = [1, 2, 3, 4, 5]
print("\n***** Absolute value of -5 is 5 ******\n")
print(abs(data))
print("\n***** Absolute value of -4.5 is 4.5 ******\n")
print(abs(data2))
print(
"\n***** Sum adds valuest together, 0 is from what value calculation starts ******\n"
)
print(sum(data3, 0))
print("\n***** Round returns value by given decimals ******\n")
value = 12.31313131313
print(round(value, 2))
print(
"\n***** Round, if return decimal amount is set to None, value will just be rounded ******\n"
)
value = 12.61313131313
print(round(value, None))
def set_plot_verbosity(new_plot_verbosity):
"""Set the plot verbosity."""
global plot_verbosity
plot_verbosity = round(new_plot_verbosity)
_check_verbosity(plot_verbosity)
def set_verbosity(new_verbosity):
"""Set the verbosity."""
global verbosity
verbosity = round(new_verbosity)
_check_verbosity(verbosity)
def to_gds(self, outfile, multiplier):
"""
Convert this label to a GDSII structure.
Parameters
----------
outfile : open file
Output to write the GDSII.
multiplier : number
A number that multiplies all dimensions written in the GDSII
structure.
"""
outfile.write(
struct.pack(
">4Hh2Hh2Hh",
4,
0x0C00,
6,
0x0D02,
self.layer,
6,
0x1602,
self.texttype,
6,
0x1701,
self.anchor,
))
if ((self.rotation is not None) or (self.magnification is not None)
or self.x_reflection):
word = 0
values = b""
if self.x_reflection:
word += 0x8000
if not (self.magnification is None):
# This flag indicates that the magnification is absolute, not
# relative (not supported).
# word += 0x0004
values += struct.pack(">2H", 12, 0x1B05) + _eight_byte_real(
self.magnification)
if not (self.rotation is None):
# This flag indicates that the rotation is absolute, not
# relative (not supported).
# word += 0x0002
values += struct.pack(">2H", 12, 0x1C05) + _eight_byte_real(
self.rotation)
outfile.write(struct.pack(">3H", 6, 0x1A01, word))
outfile.write(values)
text = self.text
if len(text) % 2 != 0:
text = text + "\0"
outfile.write(
struct.pack(
">2H2l2H",
12,
0x1003,
int(round(self.position[0] * multiplier)),
int(round(self.position[1] * multiplier)),
4 + len(text),
0x1906,
))
outfile.write(text.encode("ascii"))
outfile.write(struct.pack(">2H", 4, 0x1100))
# Question 6
from builtins import input, round, int, str
from math import *
C, H = 50, 30
def calc(D):
return sqrt((2 * C * D) / H)
D = input().split(',')
D = [int(i) for i in D]
D = [calc(i) for i in D]
D = [round(i) for i in D]
D = [str(i) for i in D]
print(",".join(D))
async def on_message(message):
if message.content.startswith(PREFIX) and not message.content.startswith(PREFIX*2):
raw_command = message.content[len(PREFIX):]
command = raw_command.split(' ')[0].lower()
raw_args = raw_command[len(command) + 1:].strip()
args = raw_args.split(' ')
if message.author.id in DEVELOPERS: # Dev only commands
if command == 'say': # Say somethin'
await send_message(message.channel, raw_args)
elif command == 'die': # Logout
await send_message(message.channel, ':wave:')
await self.logout()
elif command == 'role_ids': # DM a list of the IDs of all the roles
await send_message(message.author,
'\n'.join(['{}: {}'.format(role.name.replace('@', '@\u200b'), role.id) for role in message.guild.roles]))
await send_message(message.channel,':mailbox_with_mail:')
elif command == 'eval' and message.author.id == BOT_HOSTER:
result = None
env = {
'channel': message.channel,
'author': message.author,
'self': self,
'message': message,
'channel': message.channel,
'save_data': save_data,
}
env.update(globals())
try:
result = eval(raw_args, env)
if inspect.isawaitable(result):
result = await result
colour = 0x00FF00
except Exception as e:
result = type(e).__name__ + ': ' + str(e)
colour = 0xFF0000
embed = discord.Embed(colour=colour, title=raw_args, description='```py\n{}```'.format(result))
embed.set_author(name=message.author.display_name, icon_url=message.author.avatar_url)
try:
await message.channel.send(embed=embed)
except discord.errors.Forbidden:
pass
# General util
if command == 'help':
commands = {
'help':('[command]','get help on commands.'),
'ping':('','ping the bot.'),
'me':('','tells you about yourself.'),
'about':('','about TWOWBot.'),
'id':('','get the twow id of the current channel.'),
'prompt':('','get the prompt of the current channel.'),
'season':('','get the season of the current channel.'),
'round':('','get the round of the current channel'),
'respond':('<mtwow id> <response>','respond to a prompt.'),
'vote':('<mtwow id> [vote]','vote on a mTWOW.'),
'register':('<mtwow id>','registers the current channel with an id'),
'show_config':('[mtwow id]','get the database for a channel.'),
'responses':('[mtwow id]','get the responses for this round.'),
'set_prompt':('<prompt>','set the prompt for the current channel.'),
'start_voting':('','starts voting for the current channel.'),
'results':('[elimination]','get the results for the current mTWOW. Specify elimination amount using a number or percentage using `%`. Defaults to 20%.'),
'transfer':('<user>','transfer ownership of mtwow to someone else.'),
'delete':('','delete the current mtwow.')
}
n = len(max(list(commands.keys()), key=lambda x:len(x)))
d = '**TWOWBot help:**\n'
if not raw_args:
d += '\n'.join(['`{}{} {}` - {}'.format(PREFIX,i[0], i[1][0], i[1][1]) for i in commands.items()])
else:
while ' ' in raw_args:
raw_args = raw_args.replace(' ', ' ')
raw_args = raw_args.strip()
raw_args = raw_args.replace('\n', ' ')
passed = list(set(raw_args.split(' ')))
for i in passed:
if i in commands:
d += '`{}{} {}` - {}\n'.format(PREFIX, i, commands[i][0], commands[i][1])
else:
d += '`{}{}` - Command not found\n'.format(PREFIX, i.replace('@', '@\u200b').replace('`', '`\u200b'))
d = d[:-1]
d += '\n*Made by Bottersnike#3605, hanss314#0128 and Noahkiq#0493*'
await send_message(message.channel, d)
elif command == 'about': # Get the RickBot invite url
mess = '**This bot was developed by:**\n'
mess += 'Bottersnike#3605\n'
mess += 'hanss314#0128\n'
mess += 'Noahkiq#0493\n'
mess += '\n**This bot is being hosted by:**\n'
mess += '{}\n'.format(self.get_user(BOT_HOSTER))
#mess +='**\nTWOWBot\'s avatar by:**\n'
#mess += 'name#discrim\n'
mess += '\n**Resources:**\n'
mess += '*The official TWOWBot discord server:* https://discord.gg/eZhpeMM\n'
mess += '*Go contribute to TWOWBot on GitHub:* https://github.com/HTSTEM/TWOW_Bot\n'
mess += '*Invite TWOWBot to your server:* <https://discordapp.com/oauth2/authorize?client_id={}&scope=bot>'.format(self.user.id)
await send_message(message.channel, mess)
elif command in ['me', 'boutme', '\'boutme', 'aboutme']:
member = message.author
now = datetime.datetime.utcnow()
joined_days = now - member.joined_at
created_days = now - member.created_at
avatar = member.avatar_url
embed = discord.Embed(colour=member.colour)
embed.add_field(name='Nickname', value=member.display_name)
embed.add_field(name='User ID', value=member.id)
embed.add_field(name='Avatar', value='[Click here to show]({})'.format(avatar))
embed.add_field(name='Created', value=member.created_at.strftime('%x %X') + '\n{} days ago'.format(max(0, created_days.days)))
embed.add_field(name='Joined', value=member.joined_at.strftime('%x %X') + '\n{} days ago'.format(max(0, joined_days.days)))
roles = '\n'.join([r.mention for r in sorted(member.roles, key=lambda x:x.position, reverse=True) if r.name != '@everyone'])
if roles == '': roles = '\@everyone'
embed.add_field(name='Roles', value=roles)
embed.set_author(name=member, icon_url=avatar)
try:
await message.channel.send(embed=embed)
except discord.errors.Forbidden:
pass
elif command == 'ping':
await send_message(message.channel, 'Pong')
# TWOW-Bot ready!
elif command == 'id': # Gets the server ID used in voting
if message.channel.id in self.servers:
await send_message(message.channel,
'This mTWOW\'s identifier is `{}`'.format(self.servers[message.channel.id]))
else:
await send_message(message.channel, 'There isn\'t an entry for this mTWOW in my data. If this is an error, please contact {}.'.format(self.get_user(BOT_HOSTER)))
elif command == 'prompt': # Gets the current prompt
if message.channel.id not in self.servers:
await send_message(message.channel, 'There isn\'t an entry for this mTWOW in my data.')
return
sd = self.server_data[message.channel.id]
if 'season-{}'.format(sd['season']) not in sd['seasons']:
sd['seasons']['season-{}'.format(sd['season'])] = {}
if 'round-{}'.format(sd['round']) not in sd['seasons']['season-{}'.format(sd['season'])]['rounds']:
sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])] = {'prompt': None, 'responses': {}, 'slides': {}, 'votes': []}
round = sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])]
if round['prompt'] is None:
await send_message(message.channel, 'There\'s no prompt set yet for this mTWOW.')
return
await send_message(message.channel, 'The current prompt is:\n{}\n'.format(round['prompt'].decode('utf-8')))
elif command == 'season': # Gets the current season number
if message.channel.id not in self.servers:
await send_message(message.channel, 'There isn\'t an entry for this mTWOW in my data.')
return
sd = self.server_data[message.channel.id]
await send_message(message.channel, 'We are on season {}'.format(sd['season']))
elif command == 'round': # Get the current round number
if message.channel.id not in self.servers:
await send_message(message.channel, 'There isn\'t an entry for this mTWOW in my data.')
return
sd = self.server_data[message.channel.id]
await send_message(message.channel, 'We are on round {}'.format(sd['round']))
elif command == 'vote': # I think it makes me a hot dog. Not sure.
if not isinstance(message.channel, discord.abc.PrivateChannel):
await message.delete()
await send_message(message.channel, 'Please only vote in DMs')
return
if len(args) > 2 or not args[0]:
await send_message(message.channel,
'Usage: `{}vote <TWOW id> [vote]\nUse `.id` in the channel to get the id.'.format(PREFIX))
return
id = args[0]
s_ids = {i[1]:i[0] for i in self.servers.items()}
if id not in s_ids:
await send_message(message.channel, 'I can\'t find any mTWOW under the name `{}`.'.format(id.replace('`', '\\`')))
return
sd = self.server_data[s_ids[id]]
if not sd['voting']:
await send_message(message.channel, 'Voting hasn\'t started yet. Sorry.')
return
if 'season-{}'.format(sd['season']) not in sd['seasons']:
sd['seasons']['season-{}'.format(sd['season'])] = {}
if 'round-{}'.format(sd['round']) not in sd['seasons']['season-{}'.format(sd['season'])]['rounds']:
sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])] = {'prompt': None, 'responses': {}, 'slides': {}, 'votes': []}
round = sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])]
if len(args) == 1: # New slides needed!
if message.author.id not in round['slides']:
# Sort all responses based off their number of votes
responses = [[i, 0] for i in round['responses']]
for i in responses:
if i[0] == message.author.id:
responses.remove(i)
for vote in round['votes']:
# Each vote is a list of user IDs going from best to worst
for i in vote['vote']:
for r in responses:
if r[0] == i:
r[1] += 1
break
else:
if i != message.author.id:
responses.append([i, 1])
responses.sort(key=lambda x:x[1])
if len(responses) < 2:
await send_message(message.author, 'I don\'t have enough responses to formulate a slide. Sorry.')
return
# ~~Calculate the nubmer of responses per slide~~ Global at start of file.
# Take that many items from the list of responses.
slide = responses[:RESPONSES_PER_SLIDE]
slide = [i[0] for i in slide]
random.shuffle(slide)
# Save as a slide.
round['slides'][message.author.id] = slide
save_data()
slide = round['slides'][message.author.id]
m = '**Your slide is:**'
for n, i in enumerate(slide):
m += '\n:regional_indicator_{}: {}'.format(string.ascii_lowercase[n], round['responses'][i].decode())
if len(m) > 1500:
await send_message(message.channel,m)
m = ''
if m:
await send_message(message.channel,m)
else:
id, vote_str = raw_args.upper().split(' ')
if message.author.id not in round['slides']:
await send_message(message.author, 'You don\'t have a voting slide *to* vote on!\nUse `.vote {}` to generate one.'.format(id))
return
slide = round['slides'][message.author.id]
to = string.ascii_uppercase[len(slide) - 1]
regex = '[A-{}]{{{}}}'.format(to, len(slide))
if not re.compile(regex).match(vote_str):
await send_message(message.author, 'Please vote for **every** item on your slide exactly **once**.')
return
if len(set(vote_str)) != len(vote_str): # Check for repeats
await send_message(message.author, 'Please vote for **every** item on your slide exactly **once**.')
return
vote = list(vote_str)
for n, i in enumerate(vote):
vote[n] = slide[string.ascii_uppercase.index(i)]
round['votes'].append({
'voter': message.author.id,
'vote': vote
})
del round['slides'][message.author.id]
save_data()
await send_message(message.channel, 'Thanks for voting!')
elif command == 'respond': # Probbly handles the controlling of my kitten army
if not isinstance(message.channel, discord.abc.PrivateChannel):
await message.delete()
await send_message(message.channel, 'Please only respond in DMs')
return
if len(args) < 2:
await send_message(message.channel,
'Usage: `{}respond <TWOW id> <response>`\nUse `.id` in the channel to get the id.'.format(PREFIX))
return
id, response = raw_args.split(' ', 1)
s_ids = {i[1]:i[0] for i in self.servers.items()}
if id not in s_ids:
await send_message(message.channel, 'I can\'t find any mTWOW under the name `{}`.'.format(id.replace('`', '\\`')))
return
sd = self.server_data[s_ids[id]]
if sd['voting']:
await send_message(message.channel, 'Voting has already started. Sorry.')
return
if 'season-{}'.format(sd['season']) not in sd['seasons']:
sd['seasons']['season-{}'.format(sd['season'])] = {}
if 'round-{}'.format(sd['round']) not in sd['seasons']['season-{}'.format(sd['season'])]['rounds']:
sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])] = {'prompt': None, 'responses': {}, 'slides': {}, 'votes': []}
round = sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])]
if round['prompt'] is None:
await send_message(message.channel, 'There\'s no prompt.. How can you even?')
return
'''
for role in self.get_channel(s_ids[id]).guild.roles:
if role.id == sd['ids']['alive']:
alive_role = role
break
else:
alive_role = None'''
member = self.get_channel(s_ids[id]).guild.get_member(int(message.author.id))
if sd['round'] > 1 and message.author.id not in sd['alive']:
await send_message(message.channel, 'You are unable to submit this round. Please wait for the next season.')
return
elif sd['round'] == 1 and message.author.id not in sd['alive']:
sd['alive'].append(member.id)
if message.author.id in round['responses']:
await send_message(message.channel, '**Warning! Overwriting current response!**')
if len(response.split(' ')) > 10:
await send_message(message.channel, '**Warning! Your response looks to be over 10 words ({}).**\nThis can be ignored if you are playing a variation TWOW that doesn\'t have this limit'.format(len(response.split(' '))))
if len(response) > 140:
await send_message(message.channel, 'That is a lot of characters. Why don\'t we tone it down a bit?')
return
changed = False
with open('banned_words.txt') as bw:
banned_w = bw.read().split('\n')
for i in banned_w:
if i:
pattern = re.compile(i, re.IGNORECASE)
if pattern.match(response):
response = pattern.sub('\\*' * len(i), response)
print(response)
changed = True
if changed:
await send_message(message.channel, '**Due to rude words, your submission has been changed to:**\n{}'.format(response))
round['responses'][message.author.id] = response.encode('utf-8')
await send_message(message.channel, '**Submission recorded**')
save_data()
# TWOW owner only commands
elif command == 'start_voting':
if message.channel.id not in self.servers:
await send_message(message.channel, 'There isn\'t an entry for this mTWOW in my data.')
return
sd = self.server_data[message.channel.id]
if sd['owner'] != message.author.id:
return
if sd['voting']:
await send_message(message.channel, 'Voting is already active.')
return
roundn = sd['round']
seasonn = sd['season']
if len(sd['seasons']['season-{}'.format(seasonn)]['rounds']['round-{}'.format(roundn)]['responses']) < 2:
await send_message(message.channel, 'There aren\'t enough responses to start voting. You need at least 2.')
return
sd['voting'] = True
save_data()
await send_message(message.channel, 'Voting has been activated.')
return
elif command == 'results': # Woah? Results. Let's hope I know how to calculate these.. Haha. I didn't.
if message.channel.id not in self.servers:
await send_message(message.channel, 'There isn\'t an entry for this mTWOW in my data.')
return
sd = self.server_data[message.channel.id]
if sd['owner'] != message.author.id:
return
if not sd['voting']:
await send_message(message.channel, 'Voting hasn\'t even started yet...')
return
if 'season-{}'.format(sd['season']) not in sd['seasons']:
sd['seasons']['season-{}'.format(sd['season'])] = {}
if 'round-{}'.format(sd['round']) not in sd['seasons']['season-{}'.format(sd['season'])]['rounds']:
sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])] = {'prompt': None, 'responses': {}, 'slides': {}, 'votes': []}
round = sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])]
totals = {}
for r in round['responses']:
totals[r] = {'borda': 0, 'votes': 0, 'raw_borda': []}
vote_weights = {}
for vote in round['votes']:
if vote['voter'] not in vote_weights:
vote_weights['voter'] = 1
else:
vote_weights['voter'] += 1
for i in vote_weights:
vote_weights[i] = 1 / vote_weights[i]
for vote in round['votes']:
c = len(vote['vote'])
for n, v in enumerate(vote['vote']):
bc = c - n - 1
totals[v]['borda'] += bc
totals[v]['votes'] += 1
totals[v]['raw_borda'].append(bc / (c - 1) * 100)
totals = [{'name': i[0], **i[1]} for i in totals.items()]
def f(v):
return (v['borda'] / v['votes']) / (len(round['votes'][0]['vote']) - 1) * 100
totals.sort(key=f, reverse=True)
for twower in sd['alive']:
if twower not in round['responses']:
round['responses'][twower] = '*DNP*'.encode('UTF-8')
totals.append({'name': twower, 'borda': 0, 'votes': 1, 'raw_borda': [0]})
msg = '**Results for round {}, season {}:**'.format(sd['round'], sd['season'])
await message.delete()
await send_message(message.channel,msg)
eliminated = []
living = []
elim = int(0.8 * len(totals))
if len(args) > 0 and args[0] != '':
nums = args[0]
try:
if nums[-1] == '%':
elim = len(totals)*(100-int(nums[:-1]))//100
else:
elim = len(totals) - int(nums)
except ValueError:
await send_message(message.channel, '{} doesn\'t look like a number to me.'.format(nums))
return
for n, v in list(enumerate(totals))[::-1]:
score = f(v)
mean = sum(v['raw_borda']) / len(v['raw_borda'])
variance = sum((i - mean) ** 2 for i in v['raw_borda']) / len(v['raw_borda'])
stdev = variance ** 0.5
user = message.guild.get_member(v['name'])
if user is not None:
name = user.mention
else:
name = str(v['name'])
if str(n + 1)[-1] == '1':
if n + 1 == 11:
symbol = SUPERSCRIPT[3]
else:
symbol = SUPERSCRIPT[0]
elif str(n + 1)[-1] == '2':
if n + 1 == 12:
symbol = SUPERSCRIPT[3]
else:
symbol = SUPERSCRIPT[1]
elif str(n + 1)[-1] == '3':
if n + 1 == 13:
symbol = SUPERSCRIPT[3]
else:
symbol = SUPERSCRIPT[2]
else:
symbol = SUPERSCRIPT[3]
dead = n >= elim
if dead:
if user.id in sd['alive']:
sd['alive'].remove(user.id)
eliminated.append((name, user, v))
else:
living.append((name, user, v))
msg = '\n{}\n{} **{}{} place**: *{}*\n**{}** ({}% σ={})'.format('=' * 50, ':coffin:' if dead else ':white_check_mark:', n + 1, symbol, round['responses'][v['name']].decode('utf-8'), name, builtins.round(score, 2), builtins.round(stdev, 2))
await asyncio.sleep(len(totals) - n / 2)
await send_message(message.channel,msg)
user = message.guild.get_member(totals[0]['name'])
if user is not None:
name = user.mention
else:
name = str(v['name'])
msg = '{}\nThe winner was {}! Well done!'.format('=' * 50, name)
await send_message(message.channel,msg)
await send_message(message.channel,
'Sadly though, we have to say goodbye to {}.'.format(', '.join([i[0] for i in eliminated])))
'''
for role in message.guild.roles:
if role.id == sd['ids']['alive']:
alive_role = role
break
else:
alive_role = None
for role in message.guild.roles:
if role.id == sd['ids']['dead']:
dead_role = role
break
else:
dead_role = None'''
# Do all the round incrementing and stuff.
if len(totals) - len(eliminated) <= 1:
await send_message(message.channel,'**This season has ended! The winner was {}!**'.format(name))
sd['round'] = 1
sd['season'] += 1
else:
sd['round'] += 1
await send_message(message.channel,'**We\'re now on round {}!**'.format(sd['round']))
if 'season-{}'.format(sd['season']) not in sd['seasons']:
sd['seasons']['season-{}'.format(sd['season'])] = {'rounds':{}}
sd['alive'] = []
if 'round-{}'.format(sd['round']) not in sd['seasons']['season-{}'.format(sd['season'])]['rounds']:
sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])] = {'prompt': None, 'responses': {}, 'slides': {}, 'votes': []}
sd['voting'] = False
save_data()
# Oh yeah, and kill off dead people
'''
if alive_role is not None and dead_role is not None:
if message.guild.large:
await self.request_offline_members(message.channel)
for e in eliminated:
if e[1] is not None:
if alive_role in e[1].roles:
await e[1].remove_roles(alive_role, reason='Contestant eliminated')
if dead_role is not None:
await e[1].add_roles(dead_role, reason='Contestant eliminated')
alive_r = None
for member in message.guild.members:
if alive_r is None:
for role in member.roles:
if role.id == ALIVE_ID:
alive_r = role
if alive_r is not None and alive_r in member.roles:
for i in living:
if i[1] == member:
break
else:
await member.remove_roles(alive_r, reason='Contestant eliminated')
'''
elif command == 'responses': # List all responses this round
id = None
if len(args) > 0 and args[0] != '':
s_ids = {i[1]:i[0] for i in self.servers.items()}
if args[0] not in s_ids:
await send_message(message.channel, 'I can\'t find any mTWOW under the name `{}`.'.format(id.replace('`', '\\`')))
return
id = s_ids[args[0]]
else:
if message.channel.id not in self.servers:
await send_message(message.channel, 'There isn\'t an entry for this mTWOW in my data.')
return
id = message.channel.id
if self.server_data[id]['owner'] != message.author.id:
return
sd = self.server_data[id]
if 'season-{}'.format(sd['season']) not in sd['seasons']:
sd['seasons']['season-{}'.format(sd['season'])] = {}
if 'round-{}'.format(sd['round']) not in sd['seasons']['season-{}'.format(sd['season'])]['rounds']:
sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])] = {'prompt': None, 'responses': {}, 'slides': {}, 'votes': []}
round = sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])]
m = '**Responses for season {}, round {} so far:**'.format(sd['season'], sd['round'])
for i in round['responses'].items():
u = self.get_user(i[0])
if u is not None:
n = u.name
else:
n = i[0]
m += '\n**{}**: {}'.format(n, i[1].decode('utf-8'))
if len(m) > 1500:
await send_message(message.author,m)
m = ''
if m:
await send_message(message.author,m)
if isinstance(message.channel, discord.TextChannel):
await send_message(message.channel,':mailbox_with_mail:')
elif command == 'register': # Setup channel initially
if message.channel.id in self.servers:
owner = self.get_user(self.server_data[message.channel.id]['owner'])
if owner is not None:
await send_message(message.channel, 'This channel is already setup. The owner is {}.'.format(owner.name.replace('@', '@\u200b')))
else:
await send_message(message.channel,
'I can\'t find the owner of this mTWOW. Please contact {} to resolve this.'.format(self.get_user(BOT_HOSTER)))
else:
if not message.channel.permissions_for(message.author).manage_channels:#if user can manage that channel
return
bot_perms = message.channel.permissions_for(message.guild.get_member(self.user.id))
if not (bot_perms.send_messages and bot_perms.read_messages): #add any other perms you can think of
return
if raw_args:
if ' ' in raw_args:
await send_message(message.channel, 'No spaces in the identifier please')
return
if raw_args in list(self.servers.values()):
await send_message(message.channel, 'There\'s already a mTWOW setup with that name. Sorry.')
return
s = {}
s['owner'] = message.author.id
s['round'] = 1
s['season'] = 1
s['voting'] = False
s['alive'] = []
s['seasons'] = {'season-1':
{'rounds':
{'round-1':
{'prompt': None,
'responses': {},
'slides': {},
'votes': [],
}
}
}
}
self.server_data[message.channel.id] = s
self.servers[message.channel.id] = raw_args
save_data()
await send_message(message.channel,
'Woah! I just set up a whole mTWOW for you under the name `{}`'.format(
raw_args.replace('@', '@\u200b').replace('`', '\\`')))
else:
await send_message(message.channel, 'Usage: `{}register <short identifier>'.format(PREFIX))
elif command == 'setup': # Set congifs
pass
'''
if message.channel.id not in self.servers:
await send_message(message.channel, 'There isn\'t an entry for this server in my data.')
return
if self.server_data[message.channel.id]['owner'] != message.author.id:
return
if len(args) < 2:
await send_message(message.channel, 'Usage: `.setup <key> <value>`\nWhere key is one of `dead`, `alive`, `nts` and the value is the ID of the user or role.\nUse `.role_ids` to get the IDs of the roles.')
return
key, value = raw_args.lower().split(' ', 1)
if key not in ['dead', 'alive', 'nts']:
await send_message(message.channel, 'The key must be `dead`, `alive` or `nts`.')
return
try:
value = int(value)
except ValueError:
await send_message(message.channel, 'The value must be an id (as an integer).')
return
self.server_data[message.channel.id]['ids'][key] = value
save_data()
await send_message(message.channel, 'Wubba lubba dub dub! (Done)')
'''
elif command == 'show_config':
if message.channel.id not in self.servers:
await send_message(message.channel, 'There isn\'t an entry for this mTWOW in my data.')
return
if self.server_data[message.channel.id]['owner'] != message.author.id:
return
with open('./server_data/{}.yml'.format(message.channel.id), 'rb') as server_file:
await message.channel.send(file=discord.File(server_file))
elif command == 'set_prompt': # Summon unicorns
if message.channel.id not in self.servers:
await send_message(message.channel, 'There isn\'t an entry for this mTWOW in my data.')
return
sd = self.server_data[message.channel.id]
if sd['owner'] != message.author.id:
return
# If you're someone who likes all code to look pristine, I appologise for the next few lines :'(
if 'season-{}'.format(sd['season']) not in sd['seasons']:
sd['seasons']['season-{}'.format(sd['season'])] = {}
if 'round-{}'.format(sd['round']) not in sd['seasons']['season-{}'.format(sd['season'])]['rounds']:
sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])] = {'prompt': None, 'responses': {}, 'slides': {}, 'votes': []}
round = sd['seasons']['season-{}'.format(sd['season'])]['rounds']['round-{}'.format(sd['round'])]
if round['prompt'] is None:
round['prompt'] = raw_args.replace('@', '@\u200b').replace('`', '\\`').encode('utf-8')
save_data()
await send_message(message.channel, 'The prompt has been set to `{}` for this round.'.format(raw_args.replace('@', '@\u200b').replace('`', '\\`')))
return
else:
await send_message(message.channel, 'The prompt has been changed from `{}` to `{}` for this round.'.format(round['prompt'].decode('utf-8'), raw_args.replace('@', '@\u200b').replace('`', '\\`')))
round['prompt'] = raw_args.replace('@', '@\u200b').replace('`', '\\`').encode('utf-8')
save_data()
return
elif command == 'transfer':
if message.channel.id not in self.servers:
await send_message(message.channel, 'There isn\'t an entry for this mTWOW in my data.')
return
sd = self.server_data[message.channel.id]
if sd['owner'] != message.author.id:
return
if len(message.mentions) == 0:
await send_message(message.channel, 'Usage: `{}transfer <User>`'.format(PREFIX))
return
def check(m):
return m.channel == message.channel and m.author == message.author and m.content[0].lower() in ['y','n']
await send_message(message.channel,
'You are about to transfer your mtwow to {}. Are you 100 percent, no regrets, absolutely and completely sure about this? (y/N) Choice will default to no in 60 seconds.'.format(message.mentions[0].name))
resp = None
try:
resp = await self.wait_for('message', check=check, timeout=60)
except asyncio.TimeoutError:
await send_message(message.channel, 'Transfer Cancelled.')
return
if resp.content[0].lower() != 'y':
await send_message(message.channel, 'Transfer Cancelled.')
return
sd['owner'] = message.mentions[0].id
save_data()
await send_message(message.channel, 'MTWOW has been transfered to {}.'.format(message.mentions[0].name))
elif command == 'delete':
if message.channel.id not in self.servers:
await send_message(message.channel, 'There isn\'t an entry for this mTWOW in my data.')
return
if sd['owner'] != message.author.id:
return
def check(m):
return m.channel == message.channel and m.author == message.author and m.content[0].lower() in ['y','n']
await send_message(message.channel,
'You are about to delete your mtwow. Are you 100 percent, no regrets, absolutely and completely sure about this? (y/N) Choice will default to no in 60 seconds.')
resp = None
try:
resp = await self.wait_for('message', check=check, timeout=60)
except asyncio.TimeoutError:
await send_message(message.channel, 'Deletion Cancelled.')
return
if resp.content[0].lower() != 'y':
await send_message(message.channel, 'Deletion Cancelled.')
return
save_archive(message.channel.id)
self.servers.pop(message.channel.id, None)
self.server_data.pop(message.channel.id,None)
save_data()
await send_message(message.channel, 'MTWOW has been deleted.')
def subcellularConn(self, allCellTags, allPopTags):
from .. import sim
sim.timing('start', 'subConnectTime')
print(' Distributing synapses based on subcellular connectivity rules...')
for subConnParamTemp in list(self.params.subConnParams.values()
): # for each conn rule or parameter set
subConnParam = subConnParamTemp.copy()
# find list of pre and post cell
preCellsTags, postCellsTags = self._findPrePostCellsCondition(
allCellTags, subConnParam['preConds'], subConnParam['postConds'])
if preCellsTags and postCellsTags:
# iterate over postsyn cells to redistribute synapses
for postCellGid in postCellsTags: # for each postsyn cell
if postCellGid in self.gid2lid:
postCell = self.cells[self.gid2lid[postCellGid]]
allConns = [
conn for conn in postCell.conns
if conn['preGid'] in preCellsTags
]
if 'NetStim' in [
x['cellModel'] for x in list(preCellsTags.values())
]: # temporary fix to include netstim conns
allConns.extend([
conn for conn in postCell.conns
if conn['preGid'] == 'NetStim'
])
# group synMechs so they are not distributed separately
if 'groupSynMechs' in subConnParam:
conns = []
connsGroup = {}
#iConn = -1
for conn in allConns:
if not conn['synMech'].startswith('__grouped__'):
conns.append(conn)
#iConn = iConn + 1
connGroupLabel = '%d_%s_%.4f' % (
conn['preGid'], conn['sec'], conn['loc'])
if conn['synMech'] in subConnParam[
'groupSynMechs']:
for synMech in [
s for s in
subConnParam['groupSynMechs']
if s != conn['synMech']
]:
connGroup = next(
(c for c in allConns
if c['synMech'] == synMech
and c['sec'] == conn['sec']
and c['loc'] == conn['loc']),
None)
try:
connGroup[
'synMech'] = '__grouped__' + connGroup[
'synMech']
connsGroup[
connGroupLabel] = connGroup
except:
print(
' Warning: Grouped synMechs %s not found'
% (str(connGroup)))
else:
conns = allConns
# sort conns so reproducible across different number of cores
# use sec+preGid to avoid artificial distribution based on preGid (e.g. low gids = close to soma)
conns = sorted(conns,
key=lambda v: v['sec'] + str(v['loc']) +
str(v['preGid']))
# set sections to be used
secList = postCell._setConnSections(subConnParam)
# Uniform distribution
if subConnParam.get('density', None) == 'uniform':
# calculate new syn positions
newSecs, newLocs = postCell._distributeSynsUniformly(
secList=secList, numSyns=len(conns))
# 2D map and 1D map (radial)
elif isinstance(
subConnParam.get('density', None),
dict) and subConnParam['density']['type'] in [
'2Dmap', '1Dmap'
]:
gridY = subConnParam['density']['gridY']
gridSigma = subConnParam['density']['gridValues']
somaX, somaY, _ = self._posFromLoc(
postCell.secs['soma']['hObj'],
0.5) # get cell pos move method to Cell!
if 'fixedSomaY' in subConnParam[
'density']: # is fixed cell soma y, adjust y grid accordingly
fixedSomaY = subConnParam['density'].get(
'fixedSomaY')
gridY = [
y + (somaY - fixedSomaY) for y in gridY
] # adjust grid so cell soma is at fixedSomaY
if subConnParam['density']['type'] == '2Dmap': # 2D
gridX = [
x - somaX
for x in subConnParam['density']['gridX']
] # center x at cell soma
segNumSyn = self._interpolateSegmentSigma(
postCell, secList, gridX, gridY,
gridSigma) # move method to Cell!
elif subConnParam['density']['type'] == '1Dmap': # 1D
segNumSyn = self._interpolateSegmentSigma(
postCell, secList, None, gridY,
gridSigma) # move method to Cell!
totSyn = sum([
sum(nsyn) for nsyn in list(segNumSyn.values())
]) # summed density
scaleNumSyn = float(
len(conns)) / float(totSyn) if totSyn > 0 else 0.0
diffList = []
for sec in segNumSyn:
for seg, x in enumerate(segNumSyn[sec]):
orig = float(x * scaleNumSyn)
scaled = int(round(x * scaleNumSyn))
segNumSyn[sec][seg] = scaled
diff = orig - scaled
if diff > 0:
diffList.append([diff, sec, seg])
totSynRescale = sum(
[sum(nsyn) for nsyn in list(segNumSyn.values())])
# if missing syns due to rescaling to 0, find top values which were rounded to 0 and make 1
if totSynRescale < len(conns):
extraSyns = len(conns) - totSynRescale
diffList = sorted(diffList,
key=lambda l: l[0],
reverse=True)
for i in range(min(extraSyns, len(diffList))):
sec = diffList[i][1]
seg = diffList[i][2]
segNumSyn[sec][seg] += 1
# convert to list so can serialize and save
subConnParam['density']['gridY'] = list(
subConnParam['density']['gridY'])
subConnParam['density']['gridValues'] = list(
subConnParam['density']['gridValues'])
newSecs, newLocs = [], []
for sec, nsyns in segNumSyn.items():
for i, seg in enumerate(
postCell.secs[sec]['hObj']):
for isyn in range(nsyns[i]):
newSecs.append(sec)
newLocs.append(seg.x)
# Distance-based
elif subConnParam.get('density', None) == 'distance':
# find origin section
if 'soma' in postCell.secs:
secOrig = 'soma'
elif any([
secName.startswith('som')
for secName in list(postCell.secs.keys())
]):
secOrig = next(
secName
for secName in list(postCell.secs.keys())
if secName.startswith('soma'))
else:
secOrig = list(postCell.secs.keys())[0]
#print self.fromtodistance(postCell.secs[secOrig](0.5), postCell.secs['secs'][conn['sec']](conn['loc']))
# different case if has vs doesn't have 3d points
# h.distance(sec=h.soma[0], seg=0)
# for sec in apical:
# print h.secname()
# for seg in sec:
# print seg.x, h.distance(seg.x)
for i, (conn, newSec,
newLoc) in enumerate(zip(conns, newSecs, newLocs)):
# get conn group label before updating params
connGroupLabel = '%d_%s_%.4f' % (
conn['preGid'], conn['sec'], conn['loc'])
# update weight if weightNorm present
if 'weightNorm' in postCell.secs[
conn['sec']] and isinstance(
postCell.secs[conn['sec']]['weightNorm'],
list):
oldNseg = postCell.secs[
conn['sec']]['geom']['nseg']
oldWeightNorm = postCell.secs[
conn['sec']]['weightNorm'][
int(round(conn['loc'] * oldNseg)) - 1]
newNseg = postCell.secs[newSec]['geom']['nseg']
newWeightNorm = postCell.secs[newSec]['weightNorm'][
int(round(newLoc * newNseg)) -
1] if 'weightNorm' in postCell.secs[
newSec] else 1.0
conn['weight'] = conn[
'weight'] / oldWeightNorm * newWeightNorm
# avoid locs at 0.0 or 1.0 - triggers hoc error if syn needs an ion (eg. ca_ion)
if newLoc == 0.0: newLoc = 0.00001
elif newLoc == 1.0: newLoc = 0.99999
# updade sec and loc
conn['sec'] = newSec
conn['loc'] = newLoc
# find grouped conns
if subConnParam.get(
'groupSynMechs', None
) and conn['synMech'] in subConnParam['groupSynMechs']:
connGroup = connsGroup[
connGroupLabel] # get grouped conn from previously stored dict
connGroup['synMech'] = connGroup['synMech'].split(
'__grouped__')[1] # remove '__grouped__' label
connGroup['sec'] = newSec
connGroup['loc'] = newLoc
if newWeightNorm:
connGroup['weight'] = connGroup[
'weight'] / oldWeightNorm * newWeightNorm
sim.pc.barrier()
def convConn(self, preCellsTags, postCellsTags, connParam):
"""
Function for/to <short description of `netpyne.network.conn.convConn`>
Parameters
----------
self : <type>
<Short description of self>
**Default:** *required*
preCellsTags : <type>
<Short description of preCellsTags>
**Default:** *required*
postCellsTags : <type>
<Short description of postCellsTags>
**Default:** *required*
connParam : <type>
<Short description of connParam>
**Default:** *required*
"""
from .. import sim
if sim.cfg.verbose:
print('Generating set of convergent connections (rule: %s) ...' %
(connParam['label']))
# get list of params that have a lambda function
paramsStrFunc = [
param for param in [p + 'Func' for p in self.connStringFuncParams]
if param in connParam
]
# copy the vars into args immediately and work out which keys are associated with lambda functions only once per method
funcKeys = {}
for paramStrFunc in paramsStrFunc:
connParam[paramStrFunc + 'Args'] = connParam[paramStrFunc +
'Vars'].copy()
funcKeys[paramStrFunc] = [
key for key in connParam[paramStrFunc + 'Vars']
if callable(connParam[paramStrFunc + 'Vars'][key])
]
# converted to list only once
preCellsTagsKeys = sorted(preCellsTags)
# calculate hash for post cell gids
hashPreCells = sim.hashList(preCellsTagsKeys)
for postCellGid, postCellTags in postCellsTags.items(
): # for each postsyn cell
if postCellGid in self.gid2lid: # check if postsyn is in this node
convergence = connParam['convergenceFunc'][
postCellGid] if 'convergenceFunc' in connParam else connParam[
'convergence'] # num of presyn conns / postsyn cell
convergence = max(
min(int(round(convergence)),
len(preCellsTags) - 1), 0)
self.rand.Random123(hashPreCells, postCellGid,
sim.cfg.seeds['conn']) # init randomizer
randSample = self.randUniqueInt(self.rand, convergence + 1, 0,
len(preCellsTags) - 1)
# note: randSample[divergence] is an extra value used only if one of the random postGids coincided with the preGid
preCellsSample = {
preCellsTagsKeys[randSample[convergence]] if
preCellsTagsKeys[i] == postCellGid else preCellsTagsKeys[i]: 0
for i in randSample[0:convergence]
} # dict of selected gids of postsyn cells with removed post gid
preCellsConv = {
k: v
for k, v in preCellsTags.items() if k in preCellsSample
} # dict of selected presyn cells tags
for preCellGid, preCellTags in preCellsConv.items(
): # for each presyn cell
for paramStrFunc in paramsStrFunc: # call lambda functions to get weight func args
# update the relevant FuncArgs dict where lambda functions are known to exist in the corresponding FuncVars dict
for funcKey in funcKeys[paramStrFunc]:
connParam[paramStrFunc + 'Args'][funcKey] = connParam[
paramStrFunc + 'Vars'][funcKey](preCellTags,
postCellTags)
if preCellGid != postCellGid: # if not self-connection
self._addCellConn(connParam, preCellGid,
postCellGid) # add connection
def round(iterable, pred):
return (builtins.round(x, pred) for x in iterable)
def brevity_penalty(c, r):
if c > r:
bp = 1
else:
bp = math.exp(1-(float(r)/c))
return bp
def geometric_mean(precisions):
return (reduce(operator.mul, precisions)) ** (1.0 / len(precisions))
def BLEU(candidate, references):
precisions = []
for i in range(4):
pr, bp = count_ngram(candidate, references, i+1)
precisions.append(pr)
print('P'+str(i+1), ' = ',round(pr, 2))
print('BP = ',round(bp, 2))
bleu = geometric_mean(precisions) * bp
return bleu
if __name__ == "__main__":
candidate, references = fetch_data(sys.argv[1], sys.argv[2])
bleu = BLEU(candidate, references)
print('BLEU = ',round(bleu, 4))
out = os.open('bleu_out.txt', 'w')
out.write(str(bleu))
out.close()
#========================================================================= # This is OPEN SOURCE SOFTWARE governed by the Gnu General Public # License (GPL) version 3, as described at www.opensource.org. # Author: William H. Majoros ([email protected]) #========================================================================= from __future__ import (absolute_import, division, print_function, unicode_literals, generators, nested_scopes, with_statement) from builtins import (bytes, dict, int, list, object, range, str, ascii, chr, hex, input, next, oct, open, pow, round, super, filter, map, zip) # The above imports should allow this program to run in both Python 2 and # Python 3. You might need to update your version of module "future". import sys import ProgramName from StanParser import StanParser #========================================================================= # main() #========================================================================= if (len(sys.argv) < 3): exit(ProgramName.get() + " <infile.txt> <var1> <var2> ...\n") infile = sys.argv[1] variables = sys.argv[2:] parser = StanParser(infile) print("variable\tmedian\tSD") for var in variables: (median, mean, SD, min, max) = parser.getSummary(var) print(var, round(median, 4), round(SD, 5), sep="\t")
evaluator = BinaryClassificationEvaluator()
paraGrid = ParamGridBuilder().addGrid(rf.numTrees, num_trees).build()
cv = CrossValidator(estimator=rf,
evaluator=evaluator,
numFolds=n_fold,
estimatorParamMaps=paraGrid)
cvmodel = cv.fit(train_df)
rfpredicts = cvmodel.bestModel.transform(valid_df)
print('RandomForestClassifier')
print(cvmodel.bestModel.getNumTrees)
print(round(evaluator.evaluate(rfpredicts), n_digits))
print('')
# step 3
GBT = GBTClassifier()
evaluator = BinaryClassificationEvaluator() # areaUnderROC is default
paramGrid = ParamGridBuilder().addGrid(GBT.maxDepth, max_depth).build()
cv = CrossValidator(estimator=GBT,
evaluator=evaluator,
numFolds=n_fold,
estimatorParamMaps=paramGrid)
cvmodel = cv.fit(train_df)
def plotLFP(timeRange=None,
electrodes=['avg', 'all'],
plots=['timeSeries', 'PSD', 'spectrogram', 'locations'],
NFFT=256,
noverlap=128,
nperseg=256,
minFreq=1,
maxFreq=100,
stepFreq=1,
smooth=0,
separation=1.0,
includeAxon=True,
logx=False,
logy=False,
normSignal=False,
normPSD=False,
normSpec=False,
filtFreq=False,
filtOrder=3,
detrend=False,
transformMethod='morlet',
maxPlots=8,
overlay=False,
colors=None,
figSize=(8, 8),
fontSize=14,
lineWidth=1.5,
dpi=200,
saveData=None,
saveFig=None,
showFig=True):
"""Plots local field potentials.
Parameters
----------
timeRange : list [start, stop]
Time range to plot.
**Default:**
``None`` plots entire time range
electrodes : list
List of electrodes to include; ``'avg'`` is the average of all electrodes; ``'all'`` is each electrode separately.
**Default:** ``['avg', 'all']``
plots : list
List of plot types to show.
**Default:** ``['timeSeries', 'PSD', 'spectrogram', 'locations']``
NFFT : int (power of 2)
Number of data points used in each block for the PSD and time-freq FFT.
**Default:** ``256``
noverlap : int (<nperseg)
Number of points of overlap between segments for PSD and time-freq.
**Default:** ``128``
nperseg
Length of each segment for time-freq.
**Default:** ``256``
minFreq : float
Minimum frequency shown in plot for PSD and time-freq.
**Default:** ``1``
maxFreq : float
Maximum frequency shown in plot for PSD and time-freq.
**Default:** ``100``
stepFreq : float
Step frequency.
**Default:** ``1``
smooth : int
Window size for smoothing LFP; no smoothing if ``0``
**Default:** ``0``
separation : float
Separation factor between time-resolved LFP plots; multiplied by max LFP value.
**Default:** ``1.0``
includeAxon : bool
Whether to show the axon in the location plot.
**Default:** ``True``
logx : bool
Whether to make x-axis logarithmic
**Default:** ``False``
logy : bool
Whether to make y-axis logarithmic
**Default:** ``False``
normSignal : bool
Whether to normalize the signal.
**Default:** ``False``
normPSD : bool
Whether to normalize the power spectral density.
**Default:** ``False,
normSpec : bool
Needs documentation.
**Default:** ``False``
filtFreq : int or list
Frequency for low-pass filter (int) or frequencies for bandpass filter in a list: [low, high]
**Default:** ``False`` does not filter the data
filtOrder : int
Order of the filter defined by `filtFreq`.
**Default:** ``3``
detrend : bool
Whether to detrend.
**Default:** ``False``
transformMethod : str
Transform method.
**Default:** ``'morlet'``
**Options:** ``'fft'``
maxPlots : int
Maximum number of subplots. Currently unused.
**Default:** ``8``
overlay : bool
Whether to overlay plots or use subplots.
**Default:** ``False`` overlays plots.
colors : list
List of normalized RGB colors to use.
**Default:** ``None`` uses standard colors
figSize : list [width, height]
Size of figure in inches.
**Default:** ``(10, 8)``
fontSize : int
Font size on figure.
**Default:** ``14``
lineWidth : int
Line width.
**Default:** ``1.5``
dpi : int
Resolution of figure in dots per inch.
**Default:** ``100``
saveData : bool or str
Whether and where to save the data used to generate the plot.
**Default:** ``False``
**Options:** ``True`` autosaves the data,
``'/path/filename.ext'`` saves to a custom path and filename, valid file extensions are ``'.pkl'`` and ``'.json'``
saveFig : bool or str
Whether and where to save the figure.
**Default:** ``False``
**Options:** ``True`` autosaves the figure,
``'/path/filename.ext'`` saves to a custom path and filename, valid file extensions are ``'.png'``, ``'.jpg'``, ``'.eps'``, and ``'.tiff'``
showFig : bool
Shows the figure if ``True``.
**Default:** ``True``
Returns
-------
(figs, dict)
A tuple consisting of the matplotlib figure handles and a dictionary containing the plot data.
See Also
--------
iplotLFP :
Examples
--------
>>> import netpyne, netpyne.examples.example
>>> out = netpyne.analysis.plotLFP()
"""
# Note: should probably split funcs for signal, psd, spectrogram and locs
from .. import sim
from ..support.scalebar import add_scalebar
print('Plotting LFP ...')
if not colors: colors = colorList
# set font size
plt.rcParams.update({'font.size': fontSize})
# time range
if timeRange is None:
timeRange = [0, sim.cfg.duration]
lfp = np.array(sim.allSimData['LFP'])[
int(timeRange[0] / sim.cfg.recordStep):int(timeRange[1] /
sim.cfg.recordStep), :]
if filtFreq:
from scipy import signal
fs = 1000.0 / sim.cfg.recordStep
nyquist = fs / 2.0
if isinstance(filtFreq, list): # bandpass
Wn = [filtFreq[0] / nyquist, filtFreq[1] / nyquist]
b, a = signal.butter(filtOrder, Wn, btype='bandpass')
elif isinstance(filtFreq, Number): # lowpass
Wn = filtFreq / nyquist
b, a = signal.butter(filtOrder, Wn)
for i in range(lfp.shape[1]):
lfp[:, i] = signal.filtfilt(b, a, lfp[:, i])
if detrend:
from scipy import signal
for i in range(lfp.shape[1]):
lfp[:, i] = signal.detrend(lfp[:, i])
if normSignal:
for i in range(lfp.shape[1]):
offset = min(lfp[:, i])
if offset <= 0:
lfp[:, i] += abs(offset)
lfp[:, i] /= max(lfp[:, i])
# electrode selection
if 'all' in electrodes:
electrodes.remove('all')
electrodes.extend(list(range(int(sim.net.recXElectrode.nsites))))
# plotting
figs = []
#maxPlots = 8.0
data = {'lfp': lfp} # returned data
# time series -----------------------------------------
if 'timeSeries' in plots:
ydisp = np.absolute(lfp).max() * separation
offset = 1.0 * ydisp
t = np.arange(timeRange[0], timeRange[1], sim.cfg.recordStep)
if figSize:
figs.append(plt.figure(figsize=figSize))
for i, elec in enumerate(electrodes):
if elec == 'avg':
lfpPlot = np.mean(lfp, axis=1)
color = 'k'
lw = 1.0
elif isinstance(elec,
Number) and elec <= sim.net.recXElectrode.nsites:
lfpPlot = lfp[:, elec]
color = colors[i % len(colors)]
lw = 1.0
plt.plot(t, -lfpPlot + (i * ydisp), color=color, linewidth=lw)
if len(electrodes) > 1:
plt.text(timeRange[0] - 0.07 * (timeRange[1] - timeRange[0]),
(i * ydisp),
elec,
color=color,
ha='center',
va='top',
fontsize=fontSize,
fontweight='bold')
ax = plt.gca()
data['lfpPlot'] = lfpPlot
data['ydisp'] = ydisp
data['t'] = t
# format plot
if len(electrodes) > 1:
plt.text(timeRange[0] - 0.14 * (timeRange[1] - timeRange[0]),
(len(electrodes) * ydisp) / 2.0,
'LFP electrode',
color='k',
ha='left',
va='bottom',
fontSize=fontSize,
rotation=90)
plt.ylim(-offset, (len(electrodes)) * ydisp)
else:
plt.suptitle('LFP Signal', fontSize=fontSize, fontweight='bold')
ax.invert_yaxis()
plt.xlabel('time (ms)', fontsize=fontSize)
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['left'].set_visible(False)
plt.subplots_adjust(bottom=0.1, top=1.0, right=1.0)
# calculate scalebar size and add scalebar
round_to_n = lambda x, n, m: int(
np.ceil(round(x, -int(np.floor(np.log10(abs(x)))) +
(n - 1)) / m)) * m
scaley = 1000.0 # values in mV but want to convert to uV
m = 10.0
sizey = 100 / scaley
while sizey > 0.25 * ydisp:
try:
sizey = round_to_n(0.2 * ydisp * scaley, 1, m) / scaley
except:
sizey /= 10.0
m /= 10.0
labely = '%.3g $\mu$V' % (sizey * scaley) #)[1:]
if len(electrodes) > 1:
add_scalebar(ax,
hidey=True,
matchy=False,
hidex=False,
matchx=False,
sizex=0,
sizey=-sizey,
labely=labely,
unitsy='$\mu$V',
scaley=scaley,
loc=3,
pad=0.5,
borderpad=0.5,
sep=3,
prop=None,
barcolor="black",
barwidth=2)
else:
add_scalebar(ax,
hidey=True,
matchy=False,
hidex=True,
matchx=True,
sizex=None,
sizey=-sizey,
labely=labely,
unitsy='$\mu$V',
scaley=scaley,
unitsx='ms',
loc=3,
pad=0.5,
borderpad=0.5,
sep=3,
prop=None,
barcolor="black",
barwidth=2)
# save figure
if saveFig:
if isinstance(saveFig, basestring):
filename = saveFig
else:
filename = sim.cfg.filename + '_LFP_timeseries.png'
plt.savefig(filename, dpi=dpi)
# PSD ----------------------------------
if 'PSD' in plots:
if overlay:
figs.append(plt.figure(figsize=figSize))
else:
numCols = 1 # np.round(len(electrodes) / maxPlots) + 1
figs.append(plt.figure(figsize=(figSize[0] * numCols, figSize[1])))
#import seaborn as sb
allFreqs = []
allSignal = []
data['allFreqs'] = allFreqs
data['allSignal'] = allSignal
for i, elec in enumerate(electrodes):
if elec == 'avg':
lfpPlot = np.mean(lfp, axis=1)
elif isinstance(elec,
Number) and elec <= sim.net.recXElectrode.nsites:
lfpPlot = lfp[:, elec]
# Morlet wavelet transform method
if transformMethod == 'morlet':
from ..support.morlet import MorletSpec, index2ms
Fs = int(1000.0 / sim.cfg.recordStep)
#t_spec = np.linspace(0, index2ms(len(lfpPlot), Fs), len(lfpPlot))
morletSpec = MorletSpec(lfpPlot,
Fs,
freqmin=minFreq,
freqmax=maxFreq,
freqstep=stepFreq)
freqs = F = morletSpec.f
spec = morletSpec.TFR
signal = np.mean(spec, 1)
ylabel = 'Power'
# FFT transform method
elif transformMethod == 'fft':
Fs = int(1000.0 / sim.cfg.recordStep)
power = mlab.psd(lfpPlot,
Fs=Fs,
NFFT=NFFT,
detrend=mlab.detrend_none,
window=mlab.window_hanning,
noverlap=noverlap,
pad_to=None,
sides='default',
scale_by_freq=None)
if smooth:
signal = _smooth1d(10 * np.log10(power[0]), smooth)
else:
signal = 10 * np.log10(power[0])
freqs = power[1]
ylabel = 'Power (dB/Hz)'
allFreqs.append(freqs)
allSignal.append(signal)
# ALTERNATIVE PSD CALCULATION USING WELCH
# from http://joelyancey.com/lfp-python-practice/
# from scipy import signal as spsig
# Fs = int(1000.0/sim.cfg.recordStep)
# maxFreq=100
# f, psd = spsig.welch(lfpPlot, Fs, nperseg=100)
# plt.semilogy(f,psd,'k')
# sb.despine()
# plt.xlim((0,maxFreq))
# plt.yticks(size=fontsiz)
# plt.xticks(size=fontsiz)
# plt.ylabel('$uV^{2}/Hz$',size=fontsiz)
if normPSD:
vmax = np.max(allSignal)
for i, s in enumerate(allSignal):
allSignal[i] = allSignal[i] / vmax
for i, elec in enumerate(electrodes):
if not overlay:
plt.subplot(np.ceil(len(electrodes) / numCols), numCols, i + 1)
if elec == 'avg':
color = 'k'
elif isinstance(elec,
Number) and elec <= sim.net.recXElectrode.nsites:
color = colors[i % len(colors)]
freqs = allFreqs[i]
signal = allSignal[i]
plt.plot(freqs[freqs < maxFreq],
signal[freqs < maxFreq],
linewidth=lineWidth,
color=color,
label='Electrode %s' % (str(elec)))
plt.xlim([0, maxFreq])
if len(electrodes) > 1 and not overlay:
plt.title('Electrode %s' % (str(elec)), fontsize=fontSize)
plt.ylabel(ylabel, fontsize=fontSize)
# format plot
plt.xlabel('Frequency (Hz)', fontsize=fontSize)
if overlay:
plt.legend(fontsize=fontSize)
plt.tight_layout()
plt.suptitle('LFP Power Spectral Density',
fontsize=fontSize,
fontweight='bold') # add yaxis in opposite side
plt.subplots_adjust(bottom=0.08, top=0.92)
if logx:
pass
#from IPython import embed; embed()
# save figure
if saveFig:
if isinstance(saveFig, basestring):
filename = saveFig
else:
filename = sim.cfg.filename + '_LFP_psd.png'
plt.savefig(filename, dpi=dpi)
# Spectrogram ------------------------------
if 'spectrogram' in plots:
import matplotlib.cm as cm
numCols = 1 #np.round(len(electrodes) / maxPlots) + 1
figs.append(plt.figure(figsize=(figSize[0] * numCols, figSize[1])))
# Morlet wavelet transform method
if transformMethod == 'morlet':
from ..support.morlet import MorletSpec, index2ms
spec = []
for i, elec in enumerate(electrodes):
if elec == 'avg':
lfpPlot = np.mean(lfp, axis=1)
elif isinstance(
elec, Number) and elec <= sim.net.recXElectrode.nsites:
lfpPlot = lfp[:, elec]
fs = int(1000.0 / sim.cfg.recordStep)
t_spec = np.linspace(0, index2ms(len(lfpPlot), fs),
len(lfpPlot))
spec.append(
MorletSpec(lfpPlot,
fs,
freqmin=minFreq,
freqmax=maxFreq,
freqstep=stepFreq))
f = np.array(range(minFreq, maxFreq + 1,
stepFreq)) # only used as output for user
vmin = np.array([s.TFR for s in spec]).min()
vmax = np.array([s.TFR for s in spec]).max()
for i, elec in enumerate(electrodes):
plt.subplot(np.ceil(len(electrodes) / numCols), numCols, i + 1)
T = timeRange
F = spec[i].f
if normSpec:
spec[i].TFR = spec[i].TFR / vmax
S = spec[i].TFR
vc = [0, 1]
else:
S = spec[i].TFR
vc = [vmin, vmax]
plt.imshow(S,
extent=(np.amin(T), np.amax(T), np.amin(F),
np.amax(F)),
origin='lower',
interpolation='None',
aspect='auto',
vmin=vc[0],
vmax=vc[1],
cmap=plt.get_cmap('viridis'))
plt.colorbar(label='Power')
plt.ylabel('Hz')
plt.tight_layout()
if len(electrodes) > 1:
plt.title('Electrode %s' % (str(elec)),
fontsize=fontSize - 2)
# FFT transform method
elif transformMethod == 'fft':
from scipy import signal as spsig
spec = []
for i, elec in enumerate(electrodes):
if elec == 'avg':
lfpPlot = np.mean(lfp, axis=1)
elif isinstance(
elec, Number) and elec <= sim.net.recXElectrode.nsites:
lfpPlot = lfp[:, elec]
# creates spectrogram over a range of data
# from: http://joelyancey.com/lfp-python-practice/
fs = int(1000.0 / sim.cfg.recordStep)
f, t_spec, x_spec = spsig.spectrogram(
lfpPlot,
fs=fs,
window='hanning',
detrend=mlab.detrend_none,
nperseg=nperseg,
noverlap=noverlap,
nfft=NFFT,
mode='psd')
x_mesh, y_mesh = np.meshgrid(t_spec * 1000.0, f[f < maxFreq])
spec.append(10 * np.log10(x_spec[f < maxFreq]))
vmin = np.array(spec).min()
vmax = np.array(spec).max()
for i, elec in enumerate(electrodes):
plt.subplot(np.ceil(len(electrodes) / numCols), numCols, i + 1)
plt.pcolormesh(x_mesh,
y_mesh,
spec[i],
cmap=cm.viridis,
vmin=vmin,
vmax=vmax)
plt.colorbar(label='dB/Hz',
ticks=[np.ceil(vmin),
np.floor(vmax)])
if logy:
plt.yscale('log')
plt.ylabel('Log-frequency (Hz)')
if isinstance(logy, list):
yticks = tuple(logy)
plt.yticks(yticks, yticks)
else:
plt.ylabel('(Hz)')
if len(electrodes) > 1:
plt.title('Electrode %s' % (str(elec)),
fontsize=fontSize - 2)
plt.xlabel('time (ms)', fontsize=fontSize)
plt.tight_layout()
plt.suptitle('LFP spectrogram', size=fontSize, fontweight='bold')
plt.subplots_adjust(bottom=0.08, top=0.90)
# save figure
if saveFig:
if isinstance(saveFig, basestring):
filename = saveFig
else:
filename = sim.cfg.filename + '_LFP_timefreq.png'
plt.savefig(filename, dpi=dpi)
# locations ------------------------------
if 'locations' in plots:
cvals = [] # used to store total transfer resistance
for cell in sim.net.compartCells:
trSegs = list(
np.sum(sim.net.recXElectrode.getTransferResistance(cell.gid) *
1e3,
axis=0)) # convert from Mohm to kilohm
if not includeAxon:
i = 0
for secName, sec in cell.secs.items():
nseg = sec['hObj'].nseg #.geom.nseg
if 'axon' in secName:
for j in range(i, i + nseg):
del trSegs[j]
i += nseg
cvals.extend(trSegs)
includePost = [c.gid for c in sim.net.compartCells]
fig = sim.analysis.plotShape(includePost=includePost,
showElectrodes=electrodes,
cvals=cvals,
includeAxon=includeAxon,
dpi=dpi,
fontSize=fontSize,
saveFig=saveFig,
showFig=showFig,
figSize=figSize)[0]
figs.append(fig)
outputData = {
'LFP': lfp,
'electrodes': electrodes,
'timeRange': timeRange,
'saveData': saveData,
'saveFig': saveFig,
'showFig': showFig
}
if 'timeSeries' in plots:
outputData.update({'t': t})
if 'PSD' in plots:
outputData.update({'allFreqs': allFreqs, 'allSignal': allSignal})
if 'spectrogram' in plots:
outputData.update({
'spec': spec,
't': t_spec * 1000.0,
'freqs': f[f <= maxFreq]
})
#save figure data
if saveData:
figData = outputData
_saveFigData(figData, saveData, 'lfp')
# show fig
if showFig: _showFigure()
return figs, outputData
def plot(self, ax=None, show=False, margin=5,
linewidth=None, add_labels=False,
line2dproperties={}, xshift=0, yshift=0,
show_distances=[], print_distances=False,
virtual_atoms=True):
"""
Plots the 2D projection.
This uses modified copy-paste code by Syrtis Major (c)2014-2015
under the BSD 3-Clause license and code from matplotlib under the PSF license.
:param ax: The axes to draw to.
You can get it by calling `fig, ax=matplotlib.pyplot.subplots()`
:param show: If true, the matplotlib.pyplot.show() will be called at the
end of this function.
:param margin: A numeric value.
The margin around the plotted projection inside the (sub-)plot.
:param linewidth: The width of the lines projection.
:param add_labels: Display the name of the corresponding coarse grain element in
the middle of each segment in the projection.
Either a bool or a set of labels to display.
:param line2dproperties:
A dictionary. Will be passed as `**kwargs` to the constructor of
`matplotlib.lines.Line2D`.
See http://matplotlib.org/api/lines_api.html#matplotlib.lines.Line2D
:param xshift, yshift:
Shift the projection by the given amount inside the canvas.
:param show_distances:
A list of tuples of strings, e.g. `[("h1","h8"),("h2","m15")]`.
Show the distances between these elements in the plot
:param print_distances:
Bool. Print all distances from show_distances at the side of the plot
instead of directly next to the distance
"""
# In case of ssh without -X option, a TypeError might be raised
# during the import of pyplot
# This probably depends on the version of some library.
# This is also the reason why we import matplotlib only inside the plot function.
text = []
try:
if ax is None or show:
import matplotlib.pyplot as plt
import matplotlib.lines as lines
import matplotlib.transforms as mtransforms
import matplotlib.text as mtext
import matplotlib.font_manager as font_manager
except TypeError as e:
warnings.warn("Cannot plot projection. Maybe you could not load Gtk "
"(no X11 server available)? During the import of matplotlib"
"the following Error occured:\n {}: {}".format(type(e).__name__, e))
return
except ImportError as e:
warnings.warn("Cannot import matplotlib. Do you have matplotlib installed? "
"The following error occured:\n {}: {}".format(type(e).__name__, e))
return
# try:
# import shapely.geometry as sg
# import shapely.ops as so
# except ImportError as e:
# warnings.warn("Cannot import shapely. "
# "The following error occured:\n {}: {}".format(type(e).__name__, e))
# area=False
# #return
# else:
# area=True
area = False
polygons = []
def circles(x, y, s, c='b', ax=None, vmin=None, vmax=None, **kwargs):
"""
Make a scatter of circles plot of x vs y, where x and y are sequence
like objects of the same lengths. The size of circles are in data scale.
Parameters
----------
x,y : scalar or array_like, shape (n, )
Input data
s : scalar or array_like, shape (n, )
Radius of circle in data scale (ie. in data unit)
c : color or sequence of color, optional, default : 'b'
`c` can be a single color format string, or a sequence of color
specifications of length `N`, or a sequence of `N` numbers to be
mapped to colors using the `cmap` and `norm` specified via kwargs.
Note that `c` should not be a single numeric RGB or
RGBA sequence because that is indistinguishable from an array of
values to be colormapped. `c` can be a 2-D array in which the
rows are RGB or RGBA, however.
ax : Axes object, optional, default: None
Parent axes of the plot. It uses gca() if not specified.
vmin, vmax : scalar, optional, default: None
`vmin` and `vmax` are used in conjunction with `norm` to normalize
luminance data. If either are `None`, the min and max of the
color array is used. (Note if you pass a `norm` instance, your
settings for `vmin` and `vmax` will be ignored.)
Returns
-------
paths : `~matplotlib.collections.PathCollection`
Other parameters
----------------
kwargs : `~matplotlib.collections.Collection` properties
eg. alpha, edgecolors, facecolors, linewidths, linestyles, norm, cmap
Examples
--------
a = np.arange(11)
circles(a, a, a*0.2, c=a, alpha=0.5, edgecolor='none')
License
--------
This function is copied (and potentially modified) from
http://stackoverflow.com/a/24567352/5069869
Copyright Syrtis Major, 2014-2015
This function is under [The BSD 3-Clause License]
(http://opensource.org/licenses/BSD-3-Clause)
"""
from matplotlib.patches import Circle
from matplotlib.collections import PatchCollection
#import matplotlib.colors as colors
if ax is None:
raise TypeError()
if fus.is_string_type(c):
color = c # ie. use colors.colorConverter.to_rgba_array(c)
else:
color = None # use cmap, norm after collection is created
kwargs.update(color=color)
if np.isscalar(x):
patches = [Circle((x, y), s), ]
elif np.isscalar(s):
patches = [Circle((x_, y_), s) for x_, y_ in zip(x, y)]
else:
patches = [Circle((x_, y_), s_) for x_, y_, s_ in zip(x, y, s)]
collection = PatchCollection(patches, **kwargs)
if color is None:
collection.set_array(np.asarray(c))
if vmin is not None or vmax is not None:
collection.set_clim(vmin, vmax)
ax.add_collection(collection)
ax.autoscale_view()
return collection
class MyLine(lines.Line2D):
"""
Copied and modified from http://matplotlib.org/examples/api/line_with_text.html,
which is part of matplotlib 1.5.0 (Copyright (c) 2012-2013 Matplotlib Development
Team; All Rights Reserved).
Used under the matplotlib license: http://matplotlib.org/users/license.html
"""
def __init__(self, *args, **kwargs):
# we'll update the position when the line data is set
fm = font_manager.FontProperties(size="large", weight="demi")
self.text = mtext.Text(0, 0, '', fontproperties=fm)
lines.Line2D.__init__(self, *args, **kwargs)
# we can't access the label attr until *after* the line is
# inited
self.text.set_text(self.get_label())
def set_figure(self, figure):
self.text.set_figure(figure)
lines.Line2D.set_figure(self, figure)
def set_axes(self, axes):
self.text.set_axes(axes)
lines.Line2D.set_axes(self, axes)
def set_transform(self, transform):
# 2 pixel offset
texttrans = transform + mtransforms.Affine2D().translate(2, 2)
self.text.set_transform(texttrans)
lines.Line2D.set_transform(self, transform)
def set_data(self, x, y):
if len(x):
self.text.set_position(
((x[0] + x[-1]) / 2, (y[0] + y[-1]) / 2))
lines.Line2D.set_data(self, x, y)
def draw(self, renderer):
# draw my label at the end of the line with 2 pixel offset
lines.Line2D.draw(self, renderer)
self.text.draw(renderer)
if "linewidth" in line2dproperties and linewidth is not None:
warnings.warn(
"Got multiple values for 'linewidth' (also present in line2dproperties)")
if linewidth is not None:
line2dproperties["linewidth"] = linewidth
if "solid_capstyle" not in line2dproperties:
line2dproperties["solid_capstyle"] = "round"
if ax is None:
try:
fig, ax = plt.subplots(1, 1)
except Exception as e:
warnings.warn("Cannot create Axes or Figure. You probably have no graphical "
"display available. The Error was:\n {}: {}".format(type(e).__name__, e))
return
lprop = copy.copy(line2dproperties)
if virtual_atoms and len(self._virtual_atoms) > 0:
circles(
self._virtual_atoms[:, 0], self._virtual_atoms[:, 1], c="gray", s=0.7, ax=ax)
for label, (s, e) in self._coords.items():
if "color" not in line2dproperties:
if label.startswith("s"):
lprop["color"] = "green"
elif label.startswith("i"):
lprop["color"] = "gold"
elif label.startswith("h"):
lprop["color"] = "blue"
elif label.startswith("m"):
lprop["color"] = "red"
elif label.startswith("f") or label.startswith("t"):
lprop["color"] = "blue"
else:
lprop["color"] = "black"
if add_labels != False and (add_labels == True or label in add_labels):
lprop["label"] = label
else:
lprop["label"] = ""
#line=lines.Line2D([s[0], e[0]],[s[1],e[1]], **lprop)
line = MyLine([s[0] + xshift, e[0] + xshift],
[s[1] + yshift, e[1] + yshift], **lprop)
ax.add_line(line)
s = s + np.array([xshift, yshift])
e = e + np.array([xshift, yshift])
vec = np.array(e) - np.array(s)
nvec = np.array([vec[1], -vec[0]])
try:
div = math.sqrt(nvec[0]**2 + nvec[1]**2)
except ZeroDivisionError:
div = 100000
a = e + nvec * 5 / div
b = e - nvec * 5 / div
c = s + nvec * 5 / div
d = s - nvec * 5 / div
# For now disabling area representation
area = False
if area:
polygon = sg.Polygon([a, b, d, c])
polygons.append(polygon)
for s, e in show_distances:
st = (self._coords[s][0] + self._coords[s][1]) / 2
en = (self._coords[e][0] + self._coords[e][1]) / 2
d = ftuv.vec_distance(st, en)
if print_distances:
line = MyLine([st[0] + xshift, en[0] + xshift], [st[1] + yshift, en[1] + yshift],
color="orange", linestyle="--")
text.append("{:3} - {:3}: {:5.2f}".format(s, e, d))
else:
line = MyLine([st[0] + xshift, en[0] + xshift], [st[1] + yshift, en[1] + yshift],
label=str(round(d, 1)), color="orange", linestyle="--")
ax.add_line(line)
ax.axis(self.get_bounding_square(margin))
fm = font_manager.FontProperties(["monospace"], size="x-small")
if print_distances:
ax.text(0.01, 0.05, "\n".join(["Distances:"] + text),
transform=ax.transAxes, fontproperties=fm)
if area:
rnaArea = so.cascaded_union(polygons)
rnaXs, rnaYs = rnaArea.exterior.xy
ax.fill(rnaXs, rnaYs, alpha=0.5)
out = ax.plot()
if show:
plt.show()
return
return out
def _sample_change_from_delta(self, delta):
sample_length = self._simpler.sample.length
change_in_samples = round(old_div(delta * sample_length, 10))
return int(change_in_samples)
gradient_magnitude = math.sqrt(gradient_sum_squares)
counter = counter + 1
if gradient_magnitude < tolerance or counter > 10000:
converged = True
return(weights)
step_size = 7e-12
tolerance = 2.5e7
simple_features = ['sqft_living']
my_output = 'price'
initial_weights = np.array([-47000., 1.])
(simple_feature_matrix, output) = get_numpy_data(train_data, simple_features, my_output)
simple_weights = regression_gradient_descent(simple_feature_matrix, output,initial_weights, step_size, tolerance)
print('question 1')
print(round(simple_weights[1],1))
print('question 2 - simple model predicting house 1 price')
(test_simple_feature_matrix, test_output) = get_numpy_data(test_data, simple_features, my_output)
simpleTestOutcomes = predict_outcome(test_simple_feature_matrix, simple_weights)
print(round(simpleTestOutcomes[0]))
print('question 3 - complex model predicting house 1 price' )
model_features = ['sqft_living', 'sqft_living15']
(feature_matrix, output) = get_numpy_data(train_data, model_features,my_output)
initial_weights = np.array([-100000., 1., 1.])
step_size = 4e-12
tolerance = 1e9
model_weights = regression_gradient_descent(feature_matrix, output,initial_weights, step_size, tolerance)
def calculate_metrics(predictions,y,data_type):
start_time4 = time.time()
# Calculate ROC
evaluator = BinaryClassificationEvaluator(labelCol=y,rawPredictionCol='probability')
auroc = evaluator.evaluate(predictions,{evaluator.metricName: "areaUnderROC"})
print('AUC calculated',auroc)
selectedCols = predictions.select(F.col("probability"), F.col('prediction'), F.col(y)).rdd.map(lambda row: (float(row['probability'][1]), float(row['prediction']), float(row[y]))).collect()
y_score, y_pred, y_true = zip(*selectedCols)
# Calculate Accuracy
accuracydf=predictions.withColumn('acc',F.when(predictions.prediction==predictions[y],1).otherwise(0))
accuracydf.createOrReplaceTempView("accuracyTable")
RFaccuracy=spark.sql("select sum(acc)/count(1) as accuracy from accuracyTable").collect()[0][0]
print('Accuracy calculated',RFaccuracy)
# # Build KS Table
split1_udf = udf(lambda value: value[1].item(), DoubleType())
if data_type in ['train','valid','test','oot1','oot2']:
decileDF = predictions.select(y, split1_udf('probability').alias('probability'))
else:
decileDF = predictions.select(y, 'probability')
decileDF=decileDF.withColumn('non_target',1-decileDF[y])
window = Window.orderBy(desc("probability"))
decileDF = decileDF.withColumn("rownum", F.row_number().over(window))
decileDF.cache()
decileDF=decileDF.withColumn("rownum",decileDF["rownum"].cast("double"))
window2 = Window.orderBy("rownum")
RFbucketedData=decileDF.withColumn("deciles", F.ntile(10).over(window2))
RFbucketedData = RFbucketedData.withColumn('deciles',RFbucketedData['deciles'].cast("int"))
RFbucketedData.cache()
#a = RFbucketedData.count()
#print(RFbucketedData.show())
## to pandas from here
print('KS calculation starting')
target_cnt=RFbucketedData.groupBy('deciles').agg(F.sum(y).alias('target')).toPandas()
non_target_cnt=RFbucketedData.groupBy('deciles').agg(F.sum("non_target").alias('non_target')).toPandas()
overall_cnt=RFbucketedData.groupBy('deciles').count().alias('Total').toPandas()
overall_cnt = overall_cnt.merge(target_cnt,on='deciles',how='inner').merge(non_target_cnt,on='deciles',how='inner')
overall_cnt=overall_cnt.sort_values(by='deciles',ascending=True)
overall_cnt['Pct_target']=(overall_cnt['target']/overall_cnt['count'])*100
overall_cnt['cum_target'] = overall_cnt.target.cumsum()
overall_cnt['cum_non_target'] = overall_cnt.non_target.cumsum()
overall_cnt['%Dist_Target'] = (overall_cnt['cum_target'] / overall_cnt.target.sum())*100
overall_cnt['%Dist_non_Target'] = (overall_cnt['cum_non_target'] / overall_cnt.non_target.sum())*100
overall_cnt['spread'] = builtins.abs(overall_cnt['%Dist_Target']-overall_cnt['%Dist_non_Target'])
decile_table=overall_cnt.round(2)
print("KS_Value =", builtins.round(overall_cnt.spread.max(),2))
#print "Test Error =", builtin.round((1.0 - RFaccuracy),3)
#print "Accuracy =", builtin.round(RFaccuracy,3)
#print "AUC=", builtin.round(auroc,3)
decileDF.unpersist()
RFbucketedData.unpersist()
print("Metrics calculation process Completed in : "+ " %s seconds" % (time.time() - start_time4))
return auroc,RFaccuracy,builtins.round(overall_cnt.spread.max(),2), y_score, y_pred, y_true, overall_cnt
def round(number, *args):
"""Replacement for the built-in
:func:`round() <python:round>` function."""
return builtins.round(number, *args)
def rounded_compare(self, val1, val2):
print('Comparing {} and {} using round()'.format(val1, val2))
return builtins.round(val1) == builtins.round(val2)
def PlotAll(SaveNames,params):
from numpy import min, max, percentile,asarray,ceil,sqrt
import numpy as np
import sys
from scipy.signal import welch
from pylab import load
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib.backends.backend_pdf import PdfPages
# from scipy.ndimage.measurements import label
from AuxilaryFunctions import GetRandColors, max_intensity,SuperVoxelize,GetData,PruneComponents,SplitComponents,ThresholdShapes,MergeComponents,ThresholdData,make_sure_path_exists
# from BlockLocalNMF_AuxilaryFunctions import HALS4activity
from mpl_toolkits.axes_grid1 import make_axes_locatable
# makse sure relevant folders exist, and add to path
Results_folder='Results/'
make_sure_path_exists(Results_folder)
OASIS_path='OASIS/'
make_sure_path_exists(OASIS_path)
sys.path.append(OASIS_path)
from functions import deconvolve
## plotting params
# what to plot
plot_activities=True
plot_activities_PSD=False
plot_shapes_projections=True
plot_shapes_slices=False
plot_activityCorrs=False
plot_clustered_shape=False
plot_residual_slices=False
plot_residual_projections=False
# videos to generate
video_shapes=False
video_residual=True
video_slices=False
# what to save
save_video=True
save_plot=True
close_figs=True#close all figs right after saving (to avoid memory overload)
# PostProcessing
Split=False
Threshold=False #threshold shapes in the end and keep only connected components
Prune=False # Remove "Bad" components (where bad is defined within SplitComponent fucntion)
Merge=True # Merge highly correlated nearby components
FineTune=False # SHould we fine tune activity after post-processing? (mainly after merging)
IncludeBackground=False #should we include the background as an extracted component?
# how to plot
detrend=True #should we detrend the data (remove background component)?
scale=2 #scale colormap to enhance colors
satuartion_percentile=96 #saturate colormap ont this percentile, when ma=percentile is used
dpi=200 #for videos
restrict_support=True #in shape video, zero out data outside support of shapes
C=4 #number of components to show in shape videos (if larger then number of shapes L, then we automatically set C=L)
color_map='gray' #'gnuplot'
frame_rate=10.0 #Hz
# Fetch experimental 3D data
data=GetData(params.data_name)
if params.SuperVoxelize==True:
data=SuperVoxelize(data)
if params.ThresholdData==True:
data=ThresholdData(data)
dims=np.shape(data)
if len(dims)<4:
plot_Shapes2D=False
video_residual_2D=False
if plot_shapes_projections or plot_shapes_slices:
plot_Shapes2D=True
if video_residual==True:
video_residual_2D=True
plot_shapes_projections=False
plot_shapes_slices=False
plot_activityCorrs=False
plot_clustered_shape=False
plot_residual_slices=False
plot_residual_projections=False
video_shapes=False
video_residual=False
video_slices=False
print('2D data, ignoring 3D plots/video options')
else:
plot_Shapes2D=False
video_residual_2D=False
min_dim=np.argmin(dims[1:])
denoised_data=0
detrended_data=data
for rep in range(len(SaveNames)):
resultsName=SaveNames[rep]
try:
results=load('NMF_Results/'+SaveNames[rep])
except IOError:
if rep==0:
print('results file not found!!')
else:
break
SS=results['shapes']
AA=results['activity']
if rep>=params.Background_num:
adaptBias=False
else:
adaptBias=True
if IncludeBackground==True:
adaptBias=False
L=len(AA)-adaptBias
if L==0: #Stop if we encounter a file with zero components
break
S=SS[:-adaptBias]
b=SS[L:(L+adaptBias)]
A=AA[:-adaptBias]
f=AA[L:(L+adaptBias)]
if rep==0:
shapes=S
activity=A
background_shapes=b
background_activity=f
else:
shapes=np.append(shapes,S,axis=0)
activity=np.append(activity,A,axis=0)
background_shapes=np.append(background_shapes,b,axis=0)
background_activity=np.append(background_activity,f,axis=0)
L=len(shapes)
adaptBias=0
if Split==True:
shapes,activity,L,all_local_max=SplitComponents(shapes,activity,adaptBias)
if Merge==True:
shapes,activity,L=MergeComponents(shapes,activity,L,threshold=0.7,sig=10)
if Prune==True:
# deleted_indices=[5,9,11,14,15,17,24]+range(25,36)
shapes,activity,L=PruneComponents(shapes,activity,L,params.TargetAreaRatio)
activity_NonNegative=np.copy(activity)
activity_NonNegative[activity_NonNegative<0]=0
activity_noisy=np.copy(activity_NonNegative)
if FineTune==True:
for ll in range(L):
activity[ll], spikes, baseline, g, lam = deconvolve(activity_NonNegative[ll],optimize_g=10,penalty=0)
# activity,background_activity,S,bl,c1,sn,g,junk = update_temporal_components(data.reshape((len(data),-1)).transpose(), shapes.reshape((len(shapes),-1)).transpose(), background_shapes.reshape((len(background_shapes),-1)).transpose(), activity,background_activity,**options['temporal_params'])
activity_noisy=np.copy(activity_NonNegative)
activity_NonNegative=activity
print(str(L)+' shapes detected')
detrended_data= detrended_data - background_activity.T.dot(background_shapes.reshape((len(background_shapes), -1))).reshape(dims)
if Threshold==True:
shapes=ThresholdShapes(shapes,adaptBias,[],MaxRatio=[])
if plot_residual_projections or video_shapes or video_residual or video_slices or video_residual_2D:
colors=GetRandColors(L)
color_shapes=np.transpose(shapes.reshape(L, -1,1)*colors,[1,0,2]) #weird transpose for tensor dot product next line
denoised_data = denoised_data + (activity_NonNegative.T.dot(color_shapes)).reshape(tuple(dims)+(3,))
residual = detrended_data - activity_NonNegative.T.dot(shapes.reshape(L, -1)).reshape(dims)
if detrend==True:
data=detrended_data
#%% After loading loop - Normalize (colored) denoised data
if plot_residual_projections or video_shapes or video_residual or video_slices or video_residual_2D:
# denoised_data=denoised_data/np.max(denoised_data)
denoised_data=old_div(denoised_data,np.percentile(denoised_data[denoised_data>0],99.5)) #%% normalize denoised data range
denoised_data[denoised_data>1]=1
# plt.close('all')
#%% plotting params
ComponentsInFig=20
left = 0.05 # the left side of the subplots of the figure
right = 0.95 # the right side of the subplots of the figure
bottom = 0.05 # the bottom of the subplots of the figure
top = 0.95 # the top of the subplots of the figure
wspace = 0.1 # the amount of width reserved for blank space between subplots
hspace = 0.12 # the amount of height reserved for white space between subplots
#%% ###### Plot Individual neurons' activities
index=0 #component display index
sz=np.min([ComponentsInFig,L+adaptBias])
# a=ceil(sqrt(sz))
# b=ceil(sz/a)
a=sz
b=1
if plot_activities:
pp = PdfPages(Results_folder + 'Activities'+resultsName+'.pdf')
for ii in range(L+adaptBias):
if index==0:
# fig0=plt.figure(figsize=(dims[1] , dims[2]))
fig0=plt.figure(figsize=(11,18))
ax = plt.subplot(a,b,index+1)
# dt=1/30 # 30 Hz sample rate
time=list(range(len(activity[ii])))
plt.plot(time,activity_noisy[ii],linewidth=0.5,c='r')
plt.plot(time,activity[ii],linewidth=3,c='b')
ma=np.max([np.max(activity[ii]),np.max(activity_noisy[ii])])
plt.setp(ax, xticks=[],yticks=[0,ma])
# component number
ax.text(0.02, 0.8, str(ii),
verticalalignment='bottom', horizontalalignment='left',
transform=ax.transAxes,
color='black',weight='bold', fontsize=13)
index+=1
if ((ii%ComponentsInFig)==(ComponentsInFig-1)) or ii==(L+adaptBias-1):
index=0
if save_plot==True:
plt.subplots_adjust(left*2, bottom, right, top, wspace, hspace*2)
pp.savefig(fig0)
pp.close()
if close_figs:
plt.close('all')
#%% Plot activities` PSDs
index=0 #component display index
sz=np.min([ComponentsInFig,L+adaptBias])
a=ceil(sqrt(sz))
b=ceil(old_div(sz,a))
if plot_activities_PSD:
pp = PdfPages(Results_folder + 'ActivityPSDs'+resultsName+'.pdf')
for ii in range(L+adaptBias):
if index==0:
# fig0=plt.figure(figsize=(dims[1] , dims[2]))
fig0=plt.figure(figsize=(11,18))
ax = plt.subplot(a,b,index+1)
ff, psd_activity = welch(activity[ii], nperseg=round(old_div(len(activity[ii]), 64)))
plt.plot(ff,psd_activity,linewidth=3)
plt.setp(ax, xticks=[],yticks=[0])
# component number
ax.text(0.02, 0.8, str(ii),
verticalalignment='bottom', horizontalalignment='left',
transform=ax.transAxes,
color='black',weight='bold', fontsize=13)
index+=1
if ((ii%ComponentsInFig)==(ComponentsInFig-1)) or ii==(L+adaptBias-1):
index=0
if save_plot==True:
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
pp.savefig(fig0)
pp.close()
if close_figs:
plt.close('all')
#%% 2D shapes
index=0 #component display index
sz=np.min([ComponentsInFig,L+adaptBias])
a=ceil(0.5*sqrt(sz))
b=ceil(old_div(sz,a))
if plot_Shapes2D:
if save_plot==True:
pp = PdfPages(Results_folder + 'Shapes2D_'+resultsName+'.pdf')
for ll in range(L+adaptBias):
if index==0:
fig=plt.figure(figsize=(18 , 11))
ax = plt.subplot(a,b,index+1)
temp=shapes[ll]
mi=0
try:
ma=np.percentile(temp[temp>0],satuartion_percentile)
except IndexError:
ma=0
im=plt.imshow(temp,vmin=mi,vmax=ma,cmap=color_map)
plt.setp(ax,xticks=[],yticks=[])
mn=int(np.floor(mi)) # colorbar min value
mx=int(np.ceil(ma)) # colorbar max value
md=old_div((mx-mn),2)
# divider = make_axes_locatable(ax)
# cax = divider.append_axes("right", size="5%", pad=0.05)
# cb=plt.colorbar(im,cax=cax)
# cb.set_ticks([mn,md,mx])
# cb.set_ticklabels([mn,md,mx])
# component number
ax.text(0.02, 0.8, str(ll),
verticalalignment='bottom', horizontalalignment='left',
transform=ax.transAxes,
color='white',weight='bold', fontsize=13)
#sparsity
spar_str=str(np.round(np.mean(shapes[ll]>0)*100,2))+'%'
ax.text(0.02, 0.02, spar_str,
verticalalignment='bottom', horizontalalignment='left',
transform=ax.transAxes,
color='white',weight='bold', fontsize=13)
#L^p
for p in range(2,6,2):
Lp=old_div((np.sum(shapes[ll]**p))**(old_div(1,float(p))),np.sum(shapes[ll]))
Lp_str=str(np.round(Lp*100,2))+'%' #'L'+str(p)+'='+
ax.text(0.02+p*0.2, 0.02, Lp_str,
verticalalignment='bottom', horizontalalignment='left',
transform=ax.transAxes,
color='yellow',weight='bold', fontsize=13)
index+=1
if (ll%ComponentsInFig==(ComponentsInFig-1)) or ll==L+adaptBias-1:
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
index=0
if save_plot==True:
pp.savefig(fig)
pp.close()
if close_figs:
plt.close('all')
#%% Re-write plot code from here, so that each figure has only ComponentsInFig components
#%% ###### Plot Individual neurons' area which is correlated with their activities
a=ceil(sqrt(L+adaptBias))
b=ceil(old_div((L+adaptBias),a))
if plot_activityCorrs:
if save_plot==True:
pp = PdfPages(Results_folder + 'CorrelationWithActivity'+resultsName+'.pdf')
for dd in range(len(shapes[0].shape)):
fig0=plt.figure(figsize=(11,18))
for ii in range(L+adaptBias):
ax = plt.subplot(a,b,ii+1)
corr_imag=old_div(np.dot(activity[ii],np.transpose(data,[1,2,0,3])),np.sqrt(np.sum(data**2,axis=0)*np.sum(activity[ii]**2)))
plt.imshow(np.abs(corr_imag).max(dd),cmap=color_map)
plt.setp(ax,xticks=[],yticks=[])
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
if save_plot==True:
pp.savefig(fig0)
pp.close()
if close_figs:
plt.close('all')
#%% All Shapes projections
a=ceil(sqrt(L+adaptBias))
b=ceil(old_div((L+adaptBias),a))
if plot_shapes_projections:
if save_plot==True:
pp = PdfPages(Results_folder + 'Shapes_projections'+resultsName+'.pdf')
for dd in range(len(shapes[0].shape)):
fig=plt.figure(figsize=(18 , 11))
for ll in range(L+adaptBias):
ax = plt.subplot(a,b,ll+1)
temp=shapes[ll].max(dd)
if dd==2:
temp=temp.T
mi=np.min(shapes[ll])
ma=np.max(shapes[ll])
im=plt.imshow(temp,vmin=mi,vmax=ma,cmap=color_map)
plt.setp(ax,xticks=[],yticks=[])
mn=int(np.floor(mi)) # colorbar min value
mx=int(np.ceil(ma)) # colorbar max value
md=old_div((mx-mn),2)
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.05)
cb=plt.colorbar(im,cax=cax)
# cb.set_ticks([mn,md,mx])
# cb.set_ticklabels([mn,md,mx])
# component number
ax.text(0.02, 0.8, str(ll),
verticalalignment='bottom', horizontalalignment='left',
transform=ax.transAxes,
color='white',weight='bold', fontsize=13)
# #sparsity
spar_str=str(np.round(np.mean(shapes[ll]>0)*100,2))+'%'
ax.text(0.02, 0.02, spar_str,
verticalalignment='bottom', horizontalalignment='left',
transform=ax.transAxes,
color='white',weight='bold', fontsize=13)
# #L^p
# for p in range(2,2,2):
# Lp=(np.sum(shapes[ll]**p))**(1/float(p))/np.sum(shapes[ll])
# Lp_str=str(np.round(Lp*100,2))+'%' #'L'+str(p)+'='+
# ax.text(0.02+p*0.2, 0.02, Lp_str,
# verticalalignment='bottom', horizontalalignment='left',
# transform=ax.transAxes,
# color='yellow',weight='bold', fontsize=13)
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
if save_plot==True:
pp.savefig(fig)
pp.close()
if close_figs:
plt.close('all')
#for ll in range(L+adaptBias):
# print 'Sparsity=',np.mean(shapes[ll]>0)
#%% All Shapes slices
transpose_shape= True # should we transpose shape
ComponentsInFig=3 # number of components in Figure
index=0 #component display index
# z_slices=[0,1,2,3,4,5,6,7,8] #which z slices to look at slice plots/videos
z_slices=list(range(dims[min_dim+1])) #which z slices to look at slice plots/videos
if plot_shapes_slices:
if save_plot==True:
pp = PdfPages(Results_folder + 'Shapes_slices'+resultsName+'.pdf')
for ll in range(L+adaptBias):
if index==0:
fig=plt.figure(figsize=(18, 11))
for dd in range(len(z_slices)):
ax = plt.subplot(ComponentsInFig,len(z_slices),index*len(z_slices)+dd+1)
temp=shapes[ll].take(dd,axis=min_dim)
if transpose_shape:
temp=np.transpose(temp)
mi=np.min(shapes[ll])
ma=np.max(shapes[ll])
im=plt.imshow(temp,vmin=mi,vmax=ma,cmap=color_map)
plt.setp(ax,xticks=[],yticks=[])
if dd==0:
# component number
ax.text(0.02, 0.8, str(ll),
verticalalignment='bottom', horizontalalignment='left',
transform=ax.transAxes,
color='white',weight='bold', fontsize=13)
#sparsity
spar_str=str(np.round(np.mean(shapes[ll]>0)*100,2))+'%'
ax.text(0.02, 0.02, spar_str,
verticalalignment='bottom', horizontalalignment='left',
transform=ax.transAxes,
color='white',weight='bold', fontsize=13)
mn=int(np.floor(mi)) # colorbar min value
mx=int(np.ceil(ma)) # colorbar max value
md=old_div((mx-mn),2)
divider = make_axes_locatable(ax)
cax = divider.append_axes("bottom", size="5%", pad=0.05)
cb=plt.colorbar(im,cax=cax,orientation="horizontal")
cb.set_ticks([mn,md,mx])
cb.set_ticklabels([mn,md,mx])
#L^p
for p in range(2,2,2):
Lp=old_div((np.sum(shapes[ll]**p))**(old_div(1,float(p))),np.sum(shapes[ll]))
Lp_str=str(np.round(Lp*100,2))+'%' #'L'+str(p)+'='+
ax.text(0.02+p*0.15, 0.02, Lp_str,
verticalalignment='bottom', horizontalalignment='left',
transform=ax.transAxes,
color='yellow',weight='bold', fontsize=13)
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
index+=1
if (ll%ComponentsInFig==(ComponentsInFig-1)) or ll==L+adaptBias-1:
if save_plot==True:
pp.savefig(fig)
index=0
pp.close()
if close_figs:
plt.close('all')
#for ll in range(L+adaptBias):
# print 'Sparsity=',np.mean(shapes[ll]>0)
#%% ###### Plot Individual neurons' shape projection with clustering
a=ceil(sqrt(L+adaptBias))
b=ceil(old_div((L+adaptBias),a))
if plot_clustered_shape:
from sklearn.cluster import spectral_clustering
pp = PdfPages(Results_folder + 'ClusteredShapes'+resultsName+'.pdf')
figs=[]
for dd in range(len(shapes[0].shape)):
figs.append(plt.figure(figsize=(18 , 11)))
for ll in range(L):
ind=np.reshape(shapes[ll],(1,)+tuple(dims[1:]))>0
temp=data[np.repeat(ind,dims[0],axis=0)].reshape(dims[0],-1)
delta=1 #affinity trasnformation parameter
clust=3 #number of cluster
similarity=np.exp(old_div(-np.corrcoef(temp.T),delta))
labels = spectral_clustering(similarity, n_clusters=clust, eigen_solver='arpack')
ind2=np.array(np.nonzero(ind.reshape(-1))).reshape(-1)
temp_shape=np.repeat(np.zeros_like(shapes[ll]).reshape(-1,1),clust,axis=1)
for cc in range(clust):
temp_shape[ind2[labels==cc],cc]=1
temp_shape=temp_shape.reshape(tuple(dims[1:])+(clust,))
for dd in range(len(shapes[0].shape)):
current_fig=figs[dd]
ax = current_fig.add_subplot(a,b,ll+1)
if dd==2:
temp_shape=np.transpose(temp_shape,axes=[1,0,2,3])
ax.imshow(temp_shape.max(dd))
plt.setp(ax,xticks=[],yticks=[])
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
if save_plot==True:
for dd in range(len(shapes[0].shape)):
current_fig=figs[dd]
pp.savefig(current_fig)
pp.close()
if close_figs:
plt.close('all')
#%% ##### Video Shapes
if video_shapes:
components=list(range(min(asarray([C,L]))))
C=len(components)
if restrict_support==True:
shape_support=shapes[components[0]]>0
for cc in range(C):
shape_support=np.logical_or(shape_support,shapes[components[cc]]>0)
detrended_data=shape_support.reshape((1,)+tuple(dims[1:]))*detrended_data
fig = plt.figure(figsize=(16,7))
mi = 0
ma = max(data)*scale
#mi2 = 0
#ma2 = max(shapes[ll])*max(activity[ll])
ii=0
#import colormaps as cmaps
#cmap=cmaps.viridis
cmap=color_map
a=3
b=1+C
ax1 = plt.subplot(a,b,1)
im1 = ax1.imshow(data[ii].max(0), vmin=mi, vmax=ma,cmap=cmap)
title=ax1.set_title('Data')
#plt.colorbar(im1)
ax2=[]
ax4=[]
ax6=[]
im2=[]
im4=[]
im6=[]
for cc in range(C):
ax2.append(plt.subplot(a,b,2+cc))
comp=shapes[components[cc]].max(0)*activity_NonNegative[components[cc],ii]
ma2=max(shapes[components[cc]].max(0))*max(activity_NonNegative[components[cc]])*scale
im2.append(ax2[cc].imshow(comp,vmin=0,vmax=ma2,cmap=cmap))
#ax2[0].set_title('Shape')
# plt.colorbar(im2)
ax3 = plt.subplot(a,b,1+b)
im3 = ax3.imshow(data[ii].max(1), vmin=mi, vmax=ma,cmap=cmap)
#plt.colorbar(im3)
for cc in range(C):
ax4.append(plt.subplot(a,b,2+b+cc))
comp=shapes[components[cc]].max(1)*activity_NonNegative[components[cc],ii]
ma2=max(shapes[components[cc]].max(1))*max(activity_NonNegative[components[cc]])*scale
im4.append(ax4[cc].imshow(comp,vmin=0,vmax=ma2,cmap=cmap))
#plt.colorbar(im4)
ax5 = plt.subplot(a,b,1+2*b)
im5 = ax5.imshow(np.transpose(detrended_data[ii].max(2)), vmin=mi, vmax=ma,cmap=cmap)
#plt.colorbar(im5)
for cc in range(C):
ax6.append(plt.subplot(a,b,2+2*b+cc))
comp=np.transpose(shapes[components[cc]].max(2))*activity_NonNegative[components[cc],ii]
ma2=max(shapes[components[cc]].max(2))*max(activity_NonNegative[components[cc]])*scale
im6.append(ax6[cc].imshow(comp,vmin=0,vmax=ma2,cmap=cmap))
#plt.colorbar(im6)
fig.tight_layout()
ComponentsActive=np.array([])
for cc in range(C):
ComponentsActive=np.append(ComponentsActive,np.nonzero(activity_NonNegative[components[cc]]))
ComponentsActive=np.unique(ComponentsActive)
def update(tt):
ii=ComponentsActive[tt]
im1.set_data(data[ii].max(0))
im3.set_data(data[ii].max(1))
im5.set_data(np.transpose(data[ii].max(2)))
for cc in range(C):
im2[cc].set_data(shapes[components[cc]].max(0)*activity_NonNegative[components[cc],ii])
im4[cc].set_data(shapes[components[cc]].max(1)*activity_NonNegative[components[cc],ii])
im6[cc].set_data(np.transpose(shapes[components[cc]].max(2))*activity_NonNegative[components[cc],ii])
title.set_text('Data, time = %.1f' % ii)
if save_video==True:
writer = animation.writers['ffmpeg'](fps=10)
ani = animation.FuncAnimation(fig, update, frames=len(ComponentsActive), blit=True, repeat=False)
if restrict_support==True:
ani.save(Results_folder + 'Shapes_Restricted'+resultsName+'.mp4',dpi=dpi,writer=writer)
else:
ani.save(Results_folder + 'Shapes_'+resultsName+'.mp4',dpi=dpi,writer=writer)
else:
ani = animation.FuncAnimation(fig, update, frames=len(ComponentsActive), blit=True, repeat=False)
plt.show()
#%% ##### Plot denoised projection - Results
if plot_residual_projections==True:
dims=data.shape
cmap=color_map
pic_residual=percentile(residual, 95, axis=0)
pic_denoised = max_intensity(denoised_data, axis=0)
pic_data=percentile(data, 95, axis=0)
left = 0.05 # the left side of the subplots of the figure
right = 0.95 # the right side of the subplots of the figure
bottom = 0.05 # the bottom of the subplots of the figure
top = 0.95 # the top of the subplots of the figure
wspace = 0.05 # the amount of width reserved for blank space between subplots
hspace = 0.05 # the amount of height reserved for white space between subplots
fig1=plt.figure(figsize=(11,18))
mi=min(pic_data)
ma=max(pic_data)
ax = plt.subplot(311)
im=ax.imshow(pic_data.max(0),vmin=mi,vmax=ma,cmap=cmap)
ax.set_title('Data')
plt.colorbar(im)
plt.setp(ax,xticks=[],yticks=[])
ax2 = plt.subplot(312)
im2=ax2.imshow(max_intensity(pic_denoised,0),interpolation='None')
ax2.set_title('Denoised')
plt.setp(ax,xticks=[],yticks=[])
plt.colorbar(im2)
ax3 = plt.subplot(313)
im3=ax3.imshow(pic_residual.max(0),cmap=cmap)
ax3.set_title('Residual')
plt.setp(ax,xticks=[],yticks=[])
plt.colorbar(im3)
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
fig2=plt.figure(figsize=(11,18))
mi=min(pic_data)
ma=max(pic_data)
ax = plt.subplot(311)
im=ax.imshow(pic_data.max(1),vmin=mi,vmax=ma,cmap=cmap)
ax.set_title('Data')
plt.colorbar(im)
plt.setp(ax,xticks=[],yticks=[])
ax2 = plt.subplot(312)
im2=ax2.imshow(max_intensity(pic_denoised,1),interpolation='None')
ax2.set_title('Denoised')
plt.colorbar(im2)
plt.setp(ax,xticks=[],yticks=[])
ax3 = plt.subplot(313)
im3=ax3.imshow(pic_residual.max(1),cmap=cmap)
ax3.set_title('Residual')
plt.colorbar(im3)
plt.setp(ax,xticks=[],yticks=[])
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
fig3=plt.figure(figsize=(11,18))
mi=min(pic_data)
ma=max(pic_data)
ax = plt.subplot(311)
im=ax.imshow(pic_data.max(2).T,vmin=mi,vmax=ma,cmap=cmap)
ax.set_title('Data')
plt.colorbar(im)
plt.setp(ax,xticks=[],yticks=[])
ax2 = plt.subplot(312)
im2=ax2.imshow(np.transpose(max_intensity(pic_denoised,2),[1,0,2]),interpolation='None')
ax2.set_title('denoised')
plt.setp(ax,xticks=[],yticks=[])
plt.colorbar(im2)
ax3 = plt.subplot(313)
im3=ax3.imshow(np.transpose(pic_residual.max(2)),cmap=cmap)
ax3.set_title('Residual')
plt.colorbar(im3)
plt.setp(ax,xticks=[],yticks=[])
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
if save_plot==True:
pp = PdfPages(Results_folder + 'Data_Denoised_Residual_Projections'+resultsName+'.pdf')
pp.savefig(fig1)
pp.savefig(fig2)
pp.savefig(fig3)
pp.close()
#fig = plt.figure()
#plt.plot(MSE_array)
#plt.xlabel('Iteration')
#plt.ylabel('MSE')
#plt.show()
#%% ##### Plot denoised slices - Results
# z_slices=[0,2,4,6,8] #which z slices to look at slice plots/videos
z_slices=list(range(dims[min_dim+1])) #which z slices to look at slice plots/videos
D=len(z_slices)
if plot_residual_slices==True:
dims=data.shape
cmap=color_map
pic_residual=percentile(residual, 95, axis=0)
pic_denoised = max_intensity(denoised_data, axis=0)
pic_data=percentile(data, 95, axis=0)
a=3 #number of rows
fig1=plt.figure(figsize=(18,11))
mi=min(pic_data)
ma=max(pic_data)
for kk in range(D):
ax2 = plt.subplot(a,D,kk+1)
temp=np.squeeze(np.take(pic_denoised,(z_slices[kk],),axis=min_dim))
im2=ax2.imshow(temp,interpolation='None')
ax2.set_title('Denoised')
plt.setp(ax2,xticks=[],yticks=[])
plt.colorbar(im2)
ax = plt.subplot(a,D,kk+D+1)
temp=np.squeeze(np.take(pic_data,(z_slices[kk],),axis=min_dim))
im=ax.imshow(temp,vmin=mi,vmax=ma,cmap=cmap)
ax.set_title('Data')
plt.colorbar(im)
plt.setp(ax,xticks=[],yticks=[])
ax3 = plt.subplot(a,D,kk+2*D+1)
temp=np.squeeze(np.take(pic_residual,(z_slices[kk],),axis=min_dim))
im3=ax3.imshow(temp,cmap=cmap)
ax3.set_title('Residual')
plt.setp(ax3,xticks=[],yticks=[])
plt.colorbar(im3)
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
if save_plot==True:
pp = PdfPages(Results_folder + 'Data_Denoised_Residual_Slice_'+resultsName+'.pdf')
pp.savefig(fig1)
pp.close()
#fig = plt.figure()
#plt.plot(MSE_array)
#plt.xlabel('Iteration')
#plt.ylabel('MSE')
#plt.show()
#%% ##### 2D Video Residual
if video_residual_2D:
fig = plt.figure(figsize=(16,7))
mi = 0
ma = np.percentile(data,satuartion_percentile)
mi3 = 0
ma3 = old_div(ma,np.max([np.floor(old_div(ma,np.percentile(residual[residual>0],satuartion_percentile))),1]))
ii=0
#import colormaps as cmaps
#cmap=cmaps.viridis
cmap=color_map
a=1
b=3
im_array=[]
temp=np.shape(data[ii])
ax1 = plt.subplot(a,b,1)
pic=denoised_data[ii]
im_array += [ax1.imshow(pic,interpolation='None')]
ax1.set_title('Denoised')
plt.setp(ax1,xticks=[],yticks=[])
ax2 = plt.subplot(a,b,2)
pic=data[ii]
im_array += [ax2.imshow(pic, vmin=mi, vmax=ma,cmap=cmap)]
title=ax2.set_title('Data')
plt.setp(ax2,xticks=[],yticks=[])
divider = make_axes_locatable(ax2)
cax2 = divider.append_axes("right", size="5%", pad=0.05)
plt.colorbar(im_array[-1], cax=cax2)
ax3 = plt.subplot(a,b,3)
pic=residual[ii]
im_array += [ax3.imshow(pic, vmin=mi3, vmax=ma3,cmap=cmap)]
ax3.set_title('Residual x' + '%.1f' % (old_div(ma,ma3)))
plt.setp(ax3,xticks=[],yticks=[])
divider = make_axes_locatable(ax3)
cax3 = divider.append_axes("right", size="5%", pad=0.05)
plt.colorbar(im_array[-1], cax=cax3)
# fig.tight_layout()
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
def update(ii):
im_array[0].set_data(denoised_data[ii])
im_array[1].set_data(data[ii])
im_array[2].set_data(residual[ii])
if frame_rate!=[]:
title.set_text('Data, time = %.2f sec' % (old_div(ii,frame_rate)))
else:
title.set_text('Data, time = %.1f' % ii)
if save_video==True:
writer = animation.writers['ffmpeg'](fps=10)
ani = animation.FuncAnimation(fig, update, frames=len(data), blit=False, repeat=False)
ani.save(Results_folder + 'Data_Denoised_Residual_2D_' +resultsName+'.avi',dpi=dpi,writer=writer)
else:
ani = animation.FuncAnimation(fig, update, frames=len(data), blit=False, repeat=False)
plt.show()
#%% ##### Video Projections Residual
if video_residual:
fig = plt.figure(figsize=(16,7))
mi = 0
ma = max(data)*scale
mi3 = 0
ma3 = max(residual)*scale
ii=0
#import colormaps as cmaps
#cmap=cmaps.viridis
cmap=color_map
spatial_dims_ind=list(range(len(dims)-1))
D=len(spatial_dims_ind)
a=D
b=3
im_array=[]
transpose_flags=[]
for kk in range(D):
transpose_flags+= [False]
temp=np.shape(data[ii].max(spatial_dims_ind[kk]))
if temp[0]>temp[1]:
transpose_flags[kk]=True
for kk in range(D):
ax1 = plt.subplot(a,b,D*kk+1)
if transpose_flags[kk]==False:
pic=max_intensity(denoised_data[ii],spatial_dims_ind[kk])
else:
pic=np.transpose(max_intensity(denoised_data[ii],spatial_dims_ind[kk]),[1,0,2])
im_array += [ax1.imshow(pic,interpolation='None')]
ax1.set_title('Denoised')
plt.colorbar(im_array[-1])
plt.setp(ax1,xticks=[],yticks=[])
ax2 = plt.subplot(a,b,D*kk+2)
if transpose_flags[kk]==False:
pic=data[ii].max(spatial_dims_ind[kk])
else:
pic=np.transpose(data[ii].max(spatial_dims_ind[kk]))
im_array += [ax2.imshow(pic, vmin=mi, vmax=ma,cmap=cmap)]
title=ax2.set_title('Data')
plt.colorbar(im_array[-1])
plt.setp(ax2,xticks=[],yticks=[])
ax3 = plt.subplot(a,b,D*kk+3)
if transpose_flags[kk]==False:
pic=residual[ii].max(spatial_dims_ind[kk])
else:
pic=np.transpose(residual[ii].max(spatial_dims_ind[kk]))
im_array += [ax3.imshow(pic, vmin=mi3, vmax=ma3,cmap=cmap)]
ax3.set_title('Residual')
plt.colorbar(im_array[-1])
plt.setp(ax3,xticks=[],yticks=[])
# fig.tight_layout()
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
def update(ii):
for kk in range(D):
if transpose_flags[kk]==False:
im_array[kk*D].set_data(max_intensity(denoised_data[ii],spatial_dims_ind[kk]))
im_array[kk*D+1].set_data(data[ii].max(spatial_dims_ind[kk]))
im_array[kk*D+2].set_data(residual[ii].max(spatial_dims_ind[kk]))
else:
im_array[kk*D].set_data(np.transpose(max_intensity(denoised_data[ii],spatial_dims_ind[kk]),[1,0,2]))
im_array[kk*D+1].set_data(np.transpose(data[ii].max(spatial_dims_ind[kk])))
im_array[kk*D+2].set_data(np.transpose(residual[ii].max(spatial_dims_ind[kk])))
title.set_text('Data, time = %.1f' % ii)
if save_video==True:
writer = animation.writers['ffmpeg'](fps=10)
ani = animation.FuncAnimation(fig, update, frames=len(data), blit=False, repeat=False)
ani.save(Results_folder + 'Data_Denoised_Residual_Projections'+resultsName+'.mp4',dpi=dpi,writer=writer)
else:
ani = animation.FuncAnimation(fig, update, frames=len(data), blit=False, repeat=False)
plt.show()
#%% ##### Video Slices Residual
# z_slices=[0,2,4,6,8] #which z slices to look at slice plots/videos
z_slices=list(range(dims[min_dim+1])) #which z slices to look at slice plots/videos
if video_slices:
fig = plt.figure(figsize=(16,7))
mi = 0
ma = np.percentil(data[data>0],satuartion_percentile)
mi3 = 0
ma3 = np.percentil(data[data>0],satuartion_percentile)
ii=0
#import colormaps as cmaps
#cmap=cmaps.viridis
cmap=color_map
a=3
D=len(z_slices) #number of spatial dimensions
im_array=[]
transpose_flag= True
for kk in range(D):
ax1 = plt.subplot(a,D,kk+1)
temp=np.squeeze(np.take(denoised_data[ii],(z_slices[kk],),axis=min_dim))
if transpose_flag==False:
pic=temp
else:
pic=np.transpose(temp,[1,0,2])
im_array += [ax1.imshow(pic,interpolation='None')]
ax1.set_title('Denoised, z='+ str(z_slices[kk]+1))
plt.colorbar(im_array[-1])
plt.setp(ax1,xticks=[],yticks=[])
ax2 = plt.subplot(a,D,kk+D+1)
temp=np.squeeze(np.take(data[ii],(z_slices[kk],),axis=min_dim))
if transpose_flag==False:
pic=temp
else:
pic=np.transpose(temp)
im_array += [ax2.imshow(pic, vmin=mi, vmax=ma,cmap=cmap)]
title=ax2.set_title('Data')
plt.colorbar(im_array[-1])
plt.setp(ax2,xticks=[],yticks=[])
ax3 = plt.subplot(a,D,kk+2*D+1)
temp=np.squeeze(np.take(residual[ii],(z_slices[kk],),axis=min_dim))
if transpose_flag==False:
pic=temp
else:
pic=np.transpose(temp)
im_array += [ax3.imshow(pic, vmin=mi3, vmax=ma3,cmap=cmap)]
ax3.set_title('Residual')
plt.colorbar(im_array[-1])
plt.setp(ax3,xticks=[],yticks=[])
# fig.tight_layout()
plt.subplots_adjust(left, bottom, right, top, wspace, hspace)
def update(ii):
for kk in range(D):
temp1=np.squeeze(np.take(denoised_data[ii],(z_slices[kk],),axis=min_dim))
temp2=np.squeeze(np.take(data[ii],(z_slices[kk],),axis=min_dim))
temp3=np.squeeze(np.take(residual[ii],(z_slices[kk],),axis=min_dim))
if transpose_flag==False:
im_array[a*kk].set_data(temp1)
im_array[a*kk+1].set_data(temp2)
im_array[a*kk+2].set_data(temp3)
else:
im_array[a*kk].set_data(np.transpose(temp1,[1,0,2]))
im_array[a*kk+1].set_data(np.transpose(temp2))
im_array[a*kk+2].set_data(np.transpose(temp3))
title.set_text('Data, time = %.1f' % ii)
if save_video==True:
writer = animation.writers['ffmpeg'](fps=10)
ani = animation.FuncAnimation(fig, update, frames=len(data), blit=False, repeat=False)
ani.save(Results_folder + 'Data_Denoised_Residual_Slices'+resultsName+'.mp4',dpi=dpi,writer=writer)
else:
ani = animation.FuncAnimation(fig, update, frames=len(data), blit=False, repeat=False)
plt.show()
def test_add_integer_current_value(self, pi_point):
"""Test adding an integer to a PIPoint via the current value."""
point2 = pi_point.point + 1
assert round(point2.current_value - (pi_point.values[-1] + 1),
ndigits=7) == 0
def test_multiply_integer_two_current_value(self, pi_point):
"""Test adding an integer to a PIPoint via the current value."""
point2 = pi_point.point * 2
assert round(point2.current_value - (pi_point.values[-1] * 2),
ndigits=7) == 0
min_token_freq = 3 # 단어 최소 등장 빈도
run_start = timeit.default_timer() # 시간 측정
print('Loading raw data...')
nltk.download('brown') # nltk에서 brown 말뭉치 다운로드
# all_seqs = [['and', 'how', 'right', 'she', 'was', '.'], ['the', 'operator', 'asked', 'pityingly', '.']]
# brown 말뭉치 문장(길이가 x 이상 y이하) 추출 -> 추출된 문장의 단어들을 소문자화 후 list에 보관
# all_seqs = [['i', 'am', 'a', 'boy],['you', 'are', 'a', 'girl']...]
all_seqs = [[token.lower() for token in seq] for seq in nltk.corpus.brown.sents() if 5 <= len(seq) <= 30]
rand.shuffle(all_seqs) # all_seqs 랜덤 섞기
all_seqs = all_seqs[:50000] # all_seqs 중에 20000개만 사용
trainingset_size = round(0.9 * len(all_seqs)) # 9할을 학습데이터로
train_seqs = all_seqs[:trainingset_size] # 처음부터 trainingset_size까지를 train 데이터로
validationset_size = len(all_seqs) - trainingset_size # 1할을 검증데이터로
val_seqs = all_seqs[-validationset_size:] # 처음부터 validationset_size까지를 valid 데이터로
print('Training set size:', trainingset_size)
print('Validation set size:', validationset_size)
all_tokens = (token for seq in train_seqs for token in seq) # all_tokens = ('and', 'how', 'right', ... )
token_freqs = collections.Counter(all_tokens) # {'boy': freq, 'girl':freq, ....}
vocab = sorted(token_freqs.keys(), key=lambda token: (-token_freqs[token], token)) # 단어 빈도수 정렬
# 단어 최소 등장 빈도 이하인 단어들 모두 제거
while token_freqs[vocab[-1]] < min_token_freq:
vocab.pop()
vocab_size = len(vocab) + 2 # 문장 맨 앞, 맨 뒤 마커로 +1, 미등록어로 +1
def test_bigquery_tornadoes_it(self):
test_pipeline = TestPipeline(is_integration_test=True)
# Set extra options to the pipeline for test purpose
project = test_pipeline.get_option('project')
dataset = 'BigQueryTornadoesIT'
table = 'monthly_tornadoes_%s' % int(round(time.time() * 1000))
output_table = '.'.join([dataset, table])
query = 'SELECT month, tornado_count FROM `%s`' % output_table
pipeline_verifiers = [PipelineStateMatcher(),
BigqueryMatcher(
project=project,
query=query,
checksum=self.DEFAULT_CHECKSUM)]
extra_opts = {'output': output_table,
'on_success_matcher': all_of(*pipeline_verifiers)}
# Register cleanup before pipeline execution.
# Note that actual execution happens in reverse order.
self.addCleanup(utils.delete_bq_table, project, dataset, table)
# Get pipeline options from command argument: --test-pipeline-options,
# and start pipeline job by calling pipeline main function.
bigquery_tornadoes.run(
test_pipeline.get_full_options_as_args(**extra_opts))