def _select_mutagen_indices(self, target_index):
possible_indices = [
i for i in range(self.population_size) if i is not target_index
]
_shuffle(possible_indices)
mutagen_indices = possible_indices[:self.number_of_mutagens]
return mutagen_indices
def get_statistic_for_models(self, models, limit=100, shuffle=True):
"""
-> example of usage:
-> from myapp.models import MyModel
-> from myotherapp.models import MyOtherModel
-> from statistic.models import Statistic
-> Statistic.objects.get_statistic_for_models([MyModel, MyOtherModel], limit=50, shuffle=True)
<- [<MyModel: MyModel object>, <MyOtherModel: MyOtherModel object>, <MyModel: MyModel object>, ...]
:param models: [model_cls_A, model_cls_B]
:param limit: int, limit of total returned objects
:param shuffle: bool
:return: list of objects
"""
assert isinstance(models, list), '`models` must be list of models cls'
statistic = \
list(
chain(
map(
lambda model: self.get_statistic_for_model(model, limit=limit/len(models)),
models
)
)
)
if shuffle:
_shuffle(statistic)
return statistic
def shuffle(input_list: list) -> list:
r"""listutils.shuffle(input_list)
This function returns a shuffled list with the same elements as the input
list. Usage:
>>> alist = [0, 1, 2, 3, 4]
>>> listutils.shuffle(alist)
[2, 1, 4, 0, 3]
It differs from random.shuffle() since listutils.shuffle() outputs a new
list, preserving the input one:
>>> alist = [0, 1, 2, 3, 4]
>>> listutils.shuffle(alist)
[2, 1, 4, 0, 3]
>>> alist
[0, 1, 2, 3, 4]
>>> import random
>>> random.shuffle(alist)
>>> alist
[3, 2, 1, 4, 0]
"""
if not isinstance(input_list, list):
raise TypeError('\'input_list\' must be \'list\'')
shuffled_list = input_list[:]
_shuffle(shuffled_list)
return shuffled_list
def test_sort(self): data = list(range(2 * _getrecursionlimit())) rnddata = list(data) _shuffle(rnddata) self.assertNotEqual(data, rnddata) self.assertEqual(data, list(self.cls(rnddata))) self.assertEqual(list(reversed(data)), list(reversed(self.cls(rnddata))))
def get_statistic_for_models(self, models, limit=100, shuffle=True):
"""
-> example of usage:
-> from myapp.models import MyModel
-> from myotherapp.models import MyOtherModel
-> from statistic.models import Statistic
-> Statistic.objects.get_statistic_for_models([MyModel, MyOtherModel], limit=50, shuffle=True)
<- [<MyModel: MyModel object>, <MyOtherModel: MyOtherModel object>, <MyModel: MyModel object>, ...]
:param models: [model_cls_A, model_cls_B]
:param limit: int, limit of total returned objects
:param shuffle: bool
:return: list of objects
"""
assert isinstance(models, list), '`models` must be list of models cls'
statistic = \
list(
chain(
map(
lambda model: self.get_statistic_for_model(model, limit=limit/len(models)),
models
)
)
)
if shuffle:
_shuffle(statistic)
return statistic
def _pack_islands(self):
# Несколько итераций перепаковки
# TODO вернуть систему с раундами
boxes = list() # type: List[List[Union[float, UVTransform]]]
for metas in self._transforms.values():
for meta in metas:
boxes.append([*meta.packed_norm, meta])
_log.info("Packing {0} islands...".format(len(boxes)))
best = _int_maxsize
rounds = 15 # TODO
while rounds > 0:
rounds -= 1
# Т.к. box_pack_2d псевдослучайный и может давать несколько результатов,
# то итеративно отбираем лучшие
_shuffle(boxes)
pack_x, pack_y = _box_pack_2d(boxes)
score = max(pack_x, pack_y)
_log.info("Packing round: {0}, score: {1}...".format(rounds, score))
if score >= best:
continue
for box in boxes:
box[4].packed_norm = _Vector(tuple(box[i] / score for i in range(4)))
best = score
if best == _int_maxsize:
raise AssertionError()
def __init__(self, batch_size, image_dir, shuffle_flag):
self.images_dir = image_dir
self.files = []
self.shuffle_flag = shuffle_flag
self.num_files = 0
assert isinstance(shuffle_flag, bool)
# Check which of the entries in the top directory are themselves directories.
dirfiles = _os.listdir(image_dir)
num_imgdirs = 0
self.img_label_dict = dict(
) # storage denoting which label number corresponds with which label name.
# NOTE: the below will only grab JPG/JPEG files, and would need to be modified to grab other types of image files.
# NOTE: even if the below were modified to grab other types of image files, the DALI image processing pipeline
# uses a JPEG decoder to load in the files, so just keep that in mind.
for file in dirfiles:
# Grab the name of every entry in the top directory.
fullname = _os.path.join(image_dir, file)
# If an entry is a directory, grab the filenames for all the JPG/JPEG files in that directory.
if _os.path.isdir(fullname):
imgfiles = _os.listdir(fullname)
for imgfile in imgfiles:
thisimgpath = _os.path.join(fullname, imgfile)
if _os.path.isfile(thisimgpath) and (
thisimgpath.lower().endswith(".jpg")
or thisimgpath.lower().endswith(".jpeg")):
self.img_label_dict[thisimgpath] = num_imgdirs
self.files.append(thisimgpath)
num_imgdirs += 1
assert num_imgdirs > 0
# Store the batch size.
self.batch_size = batch_size
# Shuffle the input filenames.
if self.shuffle_flag:
_shuffle(self.files)
def test_height_after_random_set(self): data = list(range(2 * _getrecursionlimit())) size = len(data) _shuffle(data) t = self.cls() self.assertNode(t) for i in data: t._set(i, i) self.assertNode(t) self.assertLessEqual(t._root.height(), 2.0 * _log(size, 2))
def train(self, trainingSet, epochs=1, errorMin=0):
""" Trains the neural network from a training set using
backpropgation algorithm from pg 251 of Alpaydin
"""
if epochs < 1:
raise ValueError, 'epochs must be at least 1'
if errorMin < 0:
raise ValueError, 'errorMin must be greater or equal to than 0'
for epoch in range(epochs):
#Should run through the training set in random order
_shuffle(trainingSet)
for example in trainingSet:
x,r = self._transformEx(example)
Y,Z = self._propagateInput(x)
#Calc change in hidden to output weights
deltaV = [_scalorTimesVectors(self.lRate * (r[i] - Y[i]) , Z) \
for i in range(self.K)]
#Calc change in input to hidden weights
deltaW = []
for h in range(0,self.H -1):
sumError = sum([((r[i] - Y[i]) * self.V[i][h]) \
for i in range(self.K)])
scalar = self.lRate * sumError * Z[h] * (1 - Z[h])
wh = _scalorTimesVectors(scalar, x)
#now calc momentum
momentum = _scalorTimesVectors(self.mRate,self.lastDeltaW[h])
deltaW.append(_addVectors(wh,momentum))
#save deltaW for next iters momentum
self.lastDeltaW = deltaW
#apply changes to weights
self.V = _addMats(self.V,deltaV)
self.W = _addMats(self.W,deltaW)
error = self.validate(trainingSet)[0]
#if errorMin is 0 don't waste time validating
if errorMin != 0 and error <= errorMin:
return
if self.aLearning:
self._updateLearningRate(error)
def random_samples(to_sample, n):
"""computes a list of n random subsets of to_sample"""
step = len(to_sample) / n
lamb = list(to_sample) ### a helpless victim!!!
lamb.reverse() ### additional noise ;-)
_shuffle(lamb)
testing_sets = [None] * n
for i in xrange(n - 1):
testing_sets[i] = frozenset(lamb[i:i + step])
testing_sets[n - 1] = frozenset(lamb[(n - 1) * step:])
return testing_sets
def test_min_max(self): t = self.cls() self.assertNode(t) self.assertRaises(ValueError, t.min) self.assertRaises(ValueError, t.max) data = list(self.data) _shuffle(data) for item in data: t._set(item, ord(item)) self.assertNode(t) self.assertEqual(min(data), t.min()) self.assertEqual(max(data), t.max())
def __init__(self, json_file, ddir, batch_size=16, shuffle=True):
assert os.path.isdir(ddir)
super(DataGenerator, self).__init__()
self.ddir = ddir
self.batch_size = batch_size
self.examples = [json.loads(s) for s in open(json_file).readlines()]
self.batch_num = (len(self.examples) + batch_size - 1) // batch_size
with open(os.path.join(ddir, "params.json"), "r") as f:
self.ID2path = json.load(f)["ID2path"]
if shuffle:
_shuffle(self.examples)
def _new_game():
global deck, dealer_card_frame, player_card_frame
deck = list(cards)
random._shuffle(deck)
dealer_hand.clear()
player_hand.clear()
dealer_card_frame.destroy()
dealer_card_frame = tkinter.Frame(card_frame, background="green")
dealer_card_frame.grid(row=0, column=1, sticky='ew', rowspan=2)
player_card_frame.destroy()
player_card_frame = tkinter.Frame(card_frame, background="green")
player_card_frame.grid(row=2, column=1, sticky='ew', rowspan=2)
result_text.set("")
start_game()
def test_marker_func(fname, *args):
block = tosdb.TOSDB_DataBlock(N,True)
block.add_items(*ITEMS)
block.add_topics(*TOPICS)
topics = list(TOPICS)
_shuffle(topics)
for topic in topics:
item = _choice(ITEMS)
date_time = _choice([True,False])
passes = _randint(3,10)
wait = int(MAX_SEC_PER_TEST/passes)
print("+ TEST", fname, str((item,topic,date_time,passes,wait)))
test(fname, block, (item, topic, date_time) + args, passes, wait)
block.close()
def train(self, trainingSet):
_shuffle(trainingSet)
sumDeltaW = 0
sumDeltaV = 0
for example in trainingSet:
x,r = self._transformEx(example)
Y = self._initY()
Z = self._initZ()
for h in range(1,self.H):
Z[h] = _sigmoid(_mulVectors(self.W[h],x))
#print Z[h]
for i in range(0,self.K):
Y[i] = _mulVectors(self.V[i],Z)
deltaV = []
for i in range(self.K):
deltaV.append(_constTimesVectors(self.lRate * (r[i] - Y[i]) , Z))
deltaW = []
for h in range(self.H):
sumError = 0
for i in range(self.K):
sumError += (r[i] - Y[i]) * self.V[i][h]
wh = _constTimesVectors(self.lRate * sumError * Z[h] * (1 - Z[h]), x)
#now calc mommentum
momentum = _constTimesVectors(self.mRate,self.lastDeltaW[h])
deltaW.append(_addVectors(wh,momentum))
self.lastDeltaW = deltaW
for i in range(self.K):
self.V[i] = _addVectors(self.V[i], deltaV[i])
if self.V[i] == float('nan'):
self.V[i] = 0
for h in range(self.H):
self.W[h] = _addVectors(self.W[h], deltaW[h])
if self.W[h] == float('nan'):
self.W[h] = 0
sumDeltaW += _absSumMatrix(deltaW)
sumDeltaV += _absSumMatrix(deltaV)
return sumDeltaW/len(trainingSet), sumDeltaV/len(trainingSet)
def _s(l): _shuffle(l) for i in l: _shuffle(l) if isinstance(i, list): _shuffle(i) _s(i) _shuffle(l) return l
def get_data_label_pair(list_file, shuffle_data=False):
filepath_list = []
label_list = []
with open(list_file) as f:
for line in f:
filepath, label = line.split()
filepath_list.append(filepath)
label_list.append(int(label))
index_list = list(range(len(label_list)))
if shuffle_data:
c = list(zip(filepath_list, label_list, index_list))
_shuffle(c)
filepath_list, label_list, index_list = zip(*c)
with open(list_file + '.shuffle.txt', 'w') as f:
for item in c:
f.write('{} {}\n'.format(item[0], item[1]))
return filepath_list, label_list, index_list
def test_sorting(self): t = self.cls() self.assertNode(t) data = list(self.data) _shuffle(data) for i in data: t._set(i, i) self.assertNode(t) data.sort() self.assertEqual(data, list(t)) t = self.cls() self.assertNode(t) data = list(self.data) _shuffle(data) for i in data: t._set(i, i) self.assertNode(t) data.sort() self.assertEqual(list(reversed(data)), list(reversed(t)))
def load_audio(path,
with_path=True,
recursive=True,
ignore_failure=True,
random_order=False):
"""
Loads WAV file(s) from a path.
Parameters
----------
path : str
Path to WAV files to be loaded.
with_path : bool, optional
Indicates whether a path column is added to the returned SFrame.
recursive : bool, optional
Indicates whether ``load_audio`` should do a recursive directory traversal,
or only load audio files directly under ``path``.
ignore_failure : bool, optional
If True, only print warnings for failed files and keep loading the remaining
audio files.
random_order : bool, optional
Load audio files in random order.
Returns
-------
out : SFrame
Returns an SFrame with either an 'audio' column or both an 'audio' and
a 'path' column. The 'audio' column is a column of dictionaries.
Each dictionary contains two items. One item is the sample rate, in
samples per second (int type). The other item will be the data in a numpy
array. If the wav file has a single channel, the array will have a single
dimension. If there are multiple channels, the array will have shape
(L,C) where L is the number of samples and C is the number of channels.
Examples
--------
>>> audio_path = "~/Documents/myAudioFiles/"
>>> audio_sframe = tc.audio_analysis.load_audio(audio_path, recursive=True)
"""
from scipy.io import wavfile as _wavfile
all_wav_files = []
if _fnmatch(path, '*.wav'): # single file
all_wav_files.append(path)
elif recursive:
for (dir_path, _, file_names) in _os.walk(path):
for cur_file in file_names:
if _fnmatch(cur_file, '*.wav'):
all_wav_files.append(dir_path + '/' + cur_file)
else:
all_wav_files = _glob(path + '/*.wav')
if random_order:
_shuffle(all_wav_files)
result_builder = _tc.SFrameBuilder(column_types=[dict, str],
column_names=['audio', 'path'])
for cur_file_path in all_wav_files:
try:
sample_rate, data = _wavfile.read(cur_file_path)
except Exception as e:
error_string = "Could not read {}: {}".format(cur_file_path, e)
if not ignore_failure:
raise _ToolkitError(error_string)
else:
print(error_string)
continue
result_builder.append([{
'sample_rate': sample_rate,
'data': data
}, cur_file_path])
result = result_builder.close()
if not with_path:
del result['path']
return result
def __init__(self,
train_dir,
batch_size,
test_dir=None,
x_shape=(227, 227),
y_shape=None,
split=0.9,
name=None,
keep_grayscale=True,
shuffle=True):
if name is None:
self.name = os.path.basename(train_dir)
else:
self.name = name
self._x_shape = tuple(x_shape)
self._y_shape = x_shape if y_shape is None else tuple(y_shape)
self._batch_size = batch_size
# check if x is 2D
self._input_2d = len(x_shape) == 2 or x_shape[2] == 1
self._keep_grayscale = keep_grayscale
# if we're only given one dataset path (i.e. train_dir), then we need to split the dataset
if test_dir is None:
datapaths = [
os.path.join(train_dir, f) for f in os.listdir(train_dir)
]
if shuffle:
_shuffle(datapaths)
split_index = int(len(datapaths) * split)
self._training_paths = datapaths[:split_index]
self._test_paths = datapaths[split_index:]
self.num_batches = int(len(self._training_paths) / batch_size)
# if we're given two dataset paths (i.e. train_dir + test_dir) then we do NOT need to split the dataset
else:
self._training_paths = [
os.path.join(train_dir, f) for f in os.listdir(train_dir)
]
self._test_paths = [
os.path.join(test_dir, f) for f in os.listdir(test_dir)
]
if shuffle:
_shuffle(self._training_paths)
_shuffle(self._test_paths)
self.num_batches = int(len(self._training_paths) / batch_size)
self._num_training_data = len(self._training_paths)
self._num_test_data = len(self._test_paths)
self._next_training_index = 0
self._next_test_index = 0
self._training_batch_q = Queue(maxsize=5)
self._test_batch_q = Queue(maxsize=3)
self._batch_training_thread = threading.Thread(
target=self._fill_training_batch_q,
name='t-dataset-fillTrainingQueue')
self._batch_training_thread.start()
self._batch_test_thread = threading.Thread(
target=self._fill_test_batch_q, name='t-dataset-fillTestQueue')
self._batch_test_thread.start()
def shuffle(self):
self.mutex.acquire()
_shuffle(self.queue)
self.mutex.release()
def shuffle(self):
_shuffle(self._cards, random=SystemRandom().random)
def shuffle(self,rp=False): k = _dc(self) [_shuffle(k) for i in lzint(_randrange(100))] if rp: super().__init__(_dc(k)) return lzlist(k)
def shuffle(self): self.mutex.acquire() _shuffle(self.queue) self.mutex.release()
def shuffle(cls):
arr = [k + cls._cards[k] for k in cls._cards]
_shuffle(arr)
cls._cards = {i[:2]: i[2:] for i in arr}
button_frame = tkinter.Frame(mainWindow)
button_frame.grid(row=3, column=0, columnspan=3, sticky='w')
dealer_button = tkinter.Button(button_frame,
text="Dealer",
command=deal_dealer)
dealer_button.grid(row=0, column=0)
player_button = tkinter.Button(button_frame,
text="Player",
command=deal_player)
player_button.grid(row=0, column=1)
new_game_button = tkinter.Button(button_frame,
text="New Game",
command=new_game)
new_game_button.grid(row=0, column=2)
cards = []
_load_images(cards)
deck = list(cards) # creates a new, separate list
random._shuffle(deck)
dealer_hand = []
player_hand = []
start_game()
mainWindow.mainloop()
def test_int_keys_rand_order(self): for i in range(len(self.data)): data = list(self.data[:i]) _shuffle(data) self._int_keys(data)