def get_test_question_shuffled(test_question):
"""
randomly shuffles the answers for a test question
:param test_question:
:return: test_question_shuffled where
question = {question: "what?",
correct_answer_letters: ['A', ...] # may be only one
answers_shuffled: ["no", "maybe", "never", ...]
"""
answers = test_question["correct_answers"] + test_question["dummy_answers"]
answer_indices = range(len(answers))
# shuffle the indices to the answers
random_shuffle(answer_indices)
# make the randomized list of answers
answers_shuffled = [answers[indx] for indx in answer_indices]
# get the answer_letter for the correct answers
correct_answer_letters = [chr(ord('A') + answer_indices.index(letter_num))
for letter_num in range(len(test_question["correct_answers"]))]
test_question_shuffled = {"question": test_question["question"],
"correct_answer_letters": correct_answer_letters,
"answers_shuffled": answers_shuffled}
return test_question_shuffled
def on_epoch_end(self):
self._logger.info("End of epoch. Shuffling the dataset:{}".format(
ImageDataGeneration.valid_subsets.inv[self._subset]))
#Shuffle indices before iterating over the datset.
if self._randomize:
random_shuffle(self._shuffled_indices)
def play_files(self, shuffle=False):
files = self.files
if shuffle:
from random import shuffle as random_shuffle
random_shuffle(files)
for file in files:
self.play_file(file)
def __init__(self,
image_data_generator,
dataframe,
batch_size,
subset,
randomize=True):
"""It initializes the required and optional parameters
Arguments:
image_data_generator {An ImageDataGenerator object} -- A generator object that allows loading a data slice.
dataframe {A pandas.DataFrame object} -- A data frame object containing the input data.
batch_size {int} -- An integer value that indicates the size of a batch.
subset {A ImageDataSubset object} -- A ImageDataSubset value to indicate the dataset subset.
randomize {boolean} -- It indicates to randomize the dataframe.
"""
#Required parameters
self._image_data_generator = image_data_generator
self._dataframe = dataframe
self._batch_size = batch_size
self._subset = subset
self._randomize = randomize
#Internal parameters
self._dataset_size = len(dataframe)
#Randomization
self._shuffled_indices = list(range(self._dataset_size))
#Logging
self._logger = logging.get_logger(__name__)
#Pre-randomization
if self._randomize:
random_shuffle(self._shuffled_indices)
async def create_round_robin():
rr = list()
rr += [app.config.CDN_A_HOST] * app.config.CDN_A_WEIGHT
rr += [app.config.CDN_B_HOST] * app.config.CDN_B_WEIGHT
rr += ['origin'] * app.config.ORIGIN_WEIGHT
random_shuffle(rr)
return rr
def generate(length=16, uppercase=1, numbers=1, characters=1):
"""
Generates a random password of given length. Default length (16)
Arguments:
uppercase: Number of uppercase letters. Default (1)
numbers: Number of numbers. Default (1)
characters: Number of special characters. Default (1)
"""
assert (
length > (uppercase + numbers + characters)
), "length is less than the number of uppercase letters, numbers, characters combined"
lowercase_list = [
"a", "b", "c", "d", "e", "f", "g", "h", "i",
"j", "k", "l", "m", "n", "o", "p", "q", "r",
"s", "t", "u", "v", "w", "x", "y", "z"
]
uppercase_list = [
"A", "B", "C", "D", "E", "F", "G", "H", "I",
"J", "K", "L", "M", "N", "O", "P", "Q", "R",
"S", "T", "U", "V", "W", "X", "Y", "Z"
]
numbers_list = [
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"
]
characters_list = [
"!", "@", "#", "$", "%", "^", "&", "*"
]
password = []
# Add uppercase letters to password
for _u in range(uppercase):
password.append(secrets.choice(uppercase_list))
# Add numbers to password
for _n in range(numbers):
password.append(secrets.choice(numbers_list))
# Add special characters to password
for _c in range(characters):
password.append(secrets.choice(characters_list))
# Add lowercase letters to password
for _i in range(length - uppercase - numbers - characters):
password.append(secrets.choice(lowercase_list))
# Shuffle password
random_shuffle(password)
return ''.join(password)
def shuffle(self):
qpositions = list(range(0, len(self.question_ids)))
sequence = 1
random_shuffle(qpositions)
for qposition in qpositions:
self.question_ids[qposition].sequence = sequence
sequence += 1
def load_table(cls, filename, has_headers=False, separator=",", shuffle=False):
table = cls()
rows = [[val.strip() for val in line.split(separator)] for line in open(filename, 'r')]
if has_headers:
table.set_headers(rows[0])
rows = rows[1:]
if shuffle:
random_shuffle(rows)
table.set_rows(rows)
return table
def get_triplets(self, num_samples):
"""It generates a list of triplets (Anchor, Positive, Negative) for each image.
Arguments:
num_samples {int} -- The number of samples per image
Returns:
[(string, string, string)] -- A list of sample triplets
"""
self._logger.info('Parameters:: num_samples: %d', num_samples)
#Images
images = self._get_images()
#Label images
labelled_images = self._get_labelled_images()
#Triplet placeholder
samples = []
for _, label_images in tqdm(labelled_images.items(), desc = 'Generating input triplets', total = len(labelled_images)):
#Available samples
n_avail_samples = min(num_samples, len(label_images))
#Candidates for negative sample
negative_sample_candidates = images - label_images
for anchor in label_images:
#Positive samples
positive_samples = random_sample(label_images, n_avail_samples)
#Negative samples
negative_samples = random_sample(negative_sample_candidates, n_avail_samples)
#Shuffle samples
random_shuffle(positive_samples)
random_shuffle(negative_samples)
for index in range(n_avail_samples):
#Create a sample
sample = (anchor, positive_samples[index], negative_samples[index])
#Append sample to the sample list
samples.append(sample)
#Pandas DataFrame
triplets = DataFrame(samples, columns = self._output_df_cols)
return triplets
def __init__(self, str_deck=None, shuffle=True, is_small=False):
self.cards = []
if str_deck:
for str_card in str_deck.split(','):
suit, value = str_card.split('-')
self.cards.append(Card(SUIT(int(suit)), VALUE(int(value))))
else:
for suit in SUIT:
for value in VALUE:
if not is_small:
self.cards.append(Card(suit, value))
elif value >= VALUE.SIX:
self.cards.append(Card(suit, value))
if shuffle:
random_shuffle(self.cards)
def rest(base=None, gain=1.0, gap=None) -> None:
"""
snap a little bit\n
[1.3-3.6]s + base\n
:param base: base time line.
:param gain: The amplifier.
:param gap: Iterable obj but not dict, snap + random in the gap
"""
snap = [random_randint(1, 3), random_randint(1, 3), random_randint(1, 3)]
ndigit = [random_randint(1, 3), random_randint(1, 3), random_randint(1, 3)]
random_shuffle(ndigit)
choice = random_choice([0, 1, 2, 3, 4, 5])
if choice == 0:
snap = (snap[0] / (ndigit[ndigit[0] - 1]) + snap[1] /
(10**ndigit[ndigit[1] - 1]) + snap[2] /
(10**ndigit[ndigit[2] - 1]))
elif choice == 1:
snap = (snap[0] / (ndigit[ndigit[0] - 1]) + snap[2] /
(10**ndigit[ndigit[1] - 1]) + snap[1] /
(10**ndigit[ndigit[2] - 1]))
elif choice == 2:
snap = (snap[1] / (ndigit[ndigit[0] - 1]) + snap[0] /
(10**ndigit[ndigit[1] - 1]) + snap[2] /
(10**ndigit[ndigit[2] - 1]))
elif choice == 3:
snap = (snap[1] / (ndigit[ndigit[0] - 1]) + snap[2] /
(10**ndigit[ndigit[1] - 1]) + snap[0] /
(10**ndigit[ndigit[2] - 1]))
elif choice == 4:
snap = (snap[2] / (ndigit[ndigit[0] - 1]) + snap[0] /
(10**ndigit[ndigit[1] - 1]) + snap[1] /
(10**ndigit[ndigit[2] - 1]))
elif choice == 5:
snap = (snap[0] / (ndigit[ndigit[0] - 1]) + snap[1] /
(10**ndigit[ndigit[1] - 1]) + snap[0] /
(10**ndigit[ndigit[2] - 1]))
snap = round(snap, ndigit[choice % 3])
if snap < 1.5:
snap += 1
if base:
snap += int(base)
if gain != 1:
snap *= gain
if gap is not None:
snap += random_choice(gap)
Logger.log([rest, f"Now resting for [{snap}]s... "])
time_sleep(int(snap))
Logger.log([rest, "Done. "])
def createPageLock(start, end=0):
with open("./img-dl-threads/pageLock", "w", encoding="utf-8") as f:
a = []
bar = TextStatusBar(abs(start - end) * 2)
if start and not end:
for i in range(1, start + 1):
a.append(str(i))
bar.show(indentation="Creating Lock: ")
elif start and end:
for i in range(start, end + 1):
a.append(str(i))
bar.show(indentation="Creating Lock: ")
random_shuffle(a)
for i in range(len(a)):
f.write(a[i] + "\n")
bar.show(indentation="Creating Lock: ")
print()
def rest(self):
snap = [random_randint(1, 3), random_randint(1, 3), random_randint(1, 3)]
ndigit = [random_randint(1, 3), random_randint(1, 3), random_randint(1, 3)]
random_shuffle(ndigit)
choice = random_choice([0, 1, 2, 3, 4, 5])
if choice == 0:
snap = (
snap[0] / (ndigit[ndigit[0] - 1])
+ snap[1] / (10 ** ndigit[ndigit[1] - 1])
+ snap[2] / (10 ** ndigit[ndigit[2] - 1])
)
elif choice == 1:
snap = (
snap[0] / (ndigit[ndigit[0] - 1])
+ snap[2] / (10 ** ndigit[ndigit[1] - 1])
+ snap[1] / (10 ** ndigit[ndigit[2] - 1])
)
elif choice == 2:
snap = (
snap[1] / (ndigit[ndigit[0] - 1])
+ snap[0] / (10 ** ndigit[ndigit[1] - 1])
+ snap[2] / (10 ** ndigit[ndigit[2] - 1])
)
elif choice == 3:
snap = (
snap[1] / (ndigit[ndigit[0] - 1])
+ snap[2] / (10 ** ndigit[ndigit[1] - 1])
+ snap[0] / (10 ** ndigit[ndigit[2] - 1])
)
elif choice == 4:
snap = (
snap[2] / (ndigit[ndigit[0] - 1])
+ snap[0] / (10 ** ndigit[ndigit[1] - 1])
+ snap[1] / (10 ** ndigit[ndigit[2] - 1])
)
elif choice == 5:
snap = (
snap[0] / (ndigit[ndigit[0] - 1])
+ snap[1] / (10 ** ndigit[ndigit[1] - 1])
+ snap[0] / (10 ** ndigit[ndigit[2] - 1])
)
snap = round(snap, ndigit[choice % 3])
if snap < 1.5:
snap += 1
time_sleep(snap * self.x) if (self.x) else time_sleep(snap)
def place_category(self):
assert self.user
print("Generating random place-category connections")
places_categories = []
places = list(Place.objects.all())
categories = list(Category.objects.all())
random_shuffle(places)
for place in places[:-10]:
cat_nums = list(set([
random_choice(list(range(0, len(categories)))) for _ in range(1, random_choice(range(1, 3)))
]))
for cat_num in cat_nums:
connection = PlaceCategory()
connection.place = place
connection.category = categories[cat_num]
places_categories.append(connection)
PlaceCategory.objects.bulk_create(places_categories)
print("Added %s place-categories, now there are %s place-categories" % (
len(places_categories), PlaceCategory.objects.count()))
def place_article(self):
assert self.user
print("Generating random place-article connections")
place_articles = []
articles = Article.objects.all()
places = list(Place.objects.all())
place_descriptions = []
for article in articles:
random_shuffle(places)
order = 0
for place in places[:random_choice(list(range(4, 10)))]:
place_article = PlaceArticle()
place_article.place = place
place_article.article = article
order += 1
place_article.order = order
place_article.image = random_choice(list(place.placeimage_set.all()))
place_descriptions = place_descriptions if len(place_descriptions) else get_random_text(10)
place_article.description = place_descriptions.pop()
place_articles.append(place_article)
PlaceArticle.objects.bulk_create(place_articles)
print("Generated %s place-article connections" % PlaceArticle.objects.count())
def train_model_dep(nlp, dataset, parser, n_iter=15):
"""Train a given model on a given dataset for n_iter times"""
# get the labels
for _, annotation in dataset:
for dep in annotation.get("deps", []):
parser.add_label(dep)
pipe_exceptions = [
"parser", "trf_wordpiecer", "trf_tok2vec", "intent_classifier"
]
other_pipes = [
pipe for pipe in nlp.pipe_names if pipe not in pipe_exceptions
] # to be ignored
# train only the parser
with nlp.disable_pipes(*other_pipes):
optimizer = nlp.begin_training()
for _ in range(n_iter):
random_shuffle(dataset)
losses = {}
batches = minibatch(dataset, size=compounding(4.0, 32.0, 1.001))
for batch in batches:
texts, annotations = zip(*batch)
nlp.update(texts, annotations, sgd=optimizer, losses=losses)
return losses
def shuffle(self):
if self.count() < 52:
raise ValueError("Only full decks can be shuffled.")
random_shuffle(self.cards)
return self
def shuffle(self):
random_shuffle(self.cards)
def resolve_posts(self, args, context, info):
discussion_id = context.matchdict['discussion_id']
discussion = models.Discussion.get(discussion_id)
random = args.get('random', False)
Post = models.Post
related = self.get_related_posts_query(True)
# If random is True returns 10 posts, the first one is the latest post
# created by the user, then the remaining ones are in random order.
# If random is False, return all the posts in creation_date desc order.
if random:
user_id = context.authenticated_userid
if user_id is None:
first_post = None
else:
first_post = Post.query.join(
related, Post.id == related.c.post_id).filter(
Post.creator_id == user_id).order_by(
desc(Post.creation_date), Post.id).first()
query = Post.default_db.query(Post.id).join(
related, Post.id == related.c.post_id)
# retrieve ids, do the random and get the posts for these ids
post_ids = [e[0] for e in query]
limit = args.get('first', 10)
if first_post is not None:
first_post_id = first_post.id
post_ids = [
post_id for post_id in post_ids if post_id != first_post_id
]
limit -= 1
random_posts_ids = random_sample(post_ids,
min(len(post_ids), limit))
query = Post.query.filter(Post.id.in_(random_posts_ids)).options(
joinedload(Post.creator), )
if len(discussion.discussion_locales) > 1:
query = query.options(
models.LangString.subqueryload_option(Post.body))
else:
query = query.options(
models.LangString.joinedload_option(Post.body))
# The query always gives the posts in the same order.
# We need to random it again.
posts = query.all()
random_shuffle(posts)
if first_post is not None:
query = [first_post] + posts
else:
# The related query returns a list of
# (<PropositionPost id=2 >, None)
# instead of <PropositionPost id=2 > when authenticated,
# this is why we do another query here:
query = Post.query.join(
related, Post.id == related.c.post_id).filter(
Post.publication_state ==
models.PublicationStates.PUBLISHED).order_by(
desc(Post.creation_date),
Post.id).options(joinedload(Post.creator))
if len(discussion.discussion_locales) > 1:
query = query.options(
models.LangString.subqueryload_option(Post.body))
else:
query = query.options(
models.LangString.joinedload_option(Post.body))
# pagination is done after that, no need to do it ourself
return query
def resolve_posts(self, args, context, info):
discussion_id = context.matchdict['discussion_id']
discussion = models.Discussion.get(discussion_id)
random = args.get('random', False)
Post = models.Post
related = self.get_related_posts_query(True)
# If random is True returns 10 posts, the first one is the latest post
# created by the user, then the remaining ones are in random order.
# If random is False, return all the posts in creation_date desc order.
if random:
user_id = context.authenticated_userid
if user_id is None:
first_post = None
else:
first_post = Post.query.join(
related, Post.id == related.c.post_id).filter(
Post.creator_id == user_id).order_by(
desc(Post.creation_date), Post.id).first()
query = Post.default_db.query(Post.id).join(
related, Post.id == related.c.post_id)
# retrieve ids, do the random and get the posts for these ids
post_ids = [e[0] for e in query]
limit = args.get('first', 10)
if first_post is not None:
first_post_id = first_post.id
post_ids = [
post_id for post_id in post_ids if post_id != first_post_id
]
limit -= 1
random_posts_ids = random_sample(post_ids,
min(len(post_ids), limit))
query = Post.query.filter(Post.id.in_(random_posts_ids)).options(
joinedload(Post.creator), )
if len(discussion.discussion_locales) > 1:
query = query.options(
models.LangString.subqueryload_option(Post.body))
else:
query = query.options(
models.LangString.joinedload_option(Post.body))
# The query always gives the posts in the same order.
# We need to random it again.
posts = query.all()
random_shuffle(posts)
if first_post is not None:
query = [first_post] + posts
else:
# The related query returns a list of
# (<PropositionPost id=2 >, None)
# instead of <PropositionPost id=2 > when authenticated,
# this is why we do another query here:
query = Post.query.join(
related, Post.id == related.c.post_id).filter(
Post.publication_state ==
models.PublicationStates.PUBLISHED).order_by(
desc(Post.creation_date),
Post.id).options(joinedload(Post.creator))
if len(discussion.discussion_locales) > 1:
query = query.options(
models.LangString.subqueryload_option(Post.body))
else:
query = query.options(
models.LangString.joinedload_option(Post.body))
from_node = args.get('from_node')
after = args.get('after')
before = args.get('before')
# If `from_node` is specified and after/before is None or empty string,
# search the position of this node to set the `after` parameter
# which is actually `arrayconnection:position` in base64.
if from_node and not after and not before:
post_id = int(Node.from_global_id(from_node)[-1])
node_idx = len(
list(
takewhile(lambda post: post[0] != post_id,
query.with_entities(Post.id))))
args['after'] = offset_to_cursor(node_idx - 1)
# pagination is done after that, no need to do it ourself
return query
import random from random import shuffle from random import shuffle as random_shuffle fruits = ['apples', 'oranges', 'bananas'] print(fruits) random.shuffle(fruits) print(fruits) shuffle(fruits) print(fruits) random_shuffle(fruits) print(fruits) print(random.choice(fruits))
def createFileLst(dataDirs,
dataExts,
dataDim,
dataListDirs,
trainortest,
trainSetRatio=0.8,
random_seed=12345):
""" create data lists
output *.scp will be in dataListDirs
"""
dataDirs = dataDirs.split(',')
dataExts = dataExts.split(',')
dataDims = [int(x) for x in dataDim.split('_')]
assert len(dataDirs) == len(
dataExts), 'Error: sub_1_prepare_list.py dataDirs and dataExts wrong'
# get the cross-set of file lists
dataList = lstdirNoExt(dataDirs[0], dataExts[0])
for dataDir, dataExt in zip(dataDirs[1:], dataExts[1:]):
listTmp = lstdirNoExt(dataDir, dataExt)
dataList = crossSet(dataList, listTmp)
# check if file exists
if len(dataList) < 1:
display.self_print("Error: fail to generate file list. Please check:",
'error')
display.self_print(
"path_acous_feats, ext_acous_feats, path_waveform in config.py;",
'error')
display.self_print("Please also check the names of input data files.",
'error')
raise Exception("Error: fail to generate file list.")
# randomize the data file list
# sort at first,
dataList.sort()
random.seed(random_seed)
random_shuffle(dataList)
# before start, take a simple test on the configuration of feature dimension
frameNum = None
for inputDir, featDim, featName in zip(dataDirs[0:-1], dataDims[0:-1],
dataExts[0:-1]):
inputFile = os.path.join(inputDir,
dataList[0]) + '.' + featName.lstrip('.')
if os.path.isfile(inputFile):
tmpframeNum = readwrite.read_raw_mat(inputFile, featDim).shape[0]
if frameNum is None or frameNum < tmpframeNum:
frameNum = tmpframeNum
for inputDir, featDim, featName in zip(dataDirs[0:-1], dataDims[0:-1],
dataExts[0:-1]):
inputFile = os.path.join(inputDir,
dataList[0]) + '.' + featName.lstrip('.')
if os.path.isfile(inputFile):
tmpframeNum = readwrite.read_raw_mat(inputFile, featDim).shape[0]
if np.abs(frameNum - tmpframeNum) * 1.0 / frameNum > 0.1:
if featDim == readwrite.read_raw_mat(inputFile, 1).shape[0]:
pass
else:
display.self_print("Large mismatch of frame numbers %s" %
(inputFile))
display.self_print(
"Please check whether inputDim are correct", 'error')
display.self_print("Or check input features are corrupted",
'error')
raise Exception("Error: mismatch of frame numbers")
display.self_print('Generating data lists in to %s' % (dataListDirs),
'highlight')
if trainortest == 'test':
# generating for test set
display.self_print('\ttest size: %d' % (len(dataList)), 'highlight')
testFileOut = dataListDirs + os.path.sep + 'test.lst'
testFilePtr = open(testFileOut, 'w')
for fileName in dataList:
testFilePtr.write('%s\n' % (fileName))
testFilePtr.close()
else:
# determine the train and validatition set if necessary
if trainSetRatio is not None and trainSetRatio > 0.0:
if trainSetRatio < 1.0:
# a ratio
trainSetDivide = int(np.round(len(dataList) * trainSetRatio))
elif trainSetRatio < len(dataList):
# absolute value of the number of training utterances
trainSetDivide = int(trainSetRatio)
else:
# a default ratio 0.8
display.self_print(
'Warning: train_utts = 0.8 is used to divide train/val',
'warning')
trainSetDivide = int(np.round(len(dataList) * 0.8))
trainSet = dataList[0:trainSetDivide]
valSet = dataList[trainSetDivide:]
if len(valSet) > len(trainSet):
display.self_print(
"Warning: validation set is larger than training set",
'warning')
display.self_print(
"It's better to change train_utts in config.py", 'warning')
trainFileOut = dataListDirs + os.path.sep + 'train.lst'
trainFilePtr = open(trainFileOut, 'w')
for fileName in trainSet:
trainFilePtr.write('%s\n' % (fileName))
trainFilePtr.close()
if len(valSet):
valFileOut = dataListDirs + os.path.sep + 'val.lst'
valFilePtr = open(valFileOut, 'w')
for fileName in valSet:
valFilePtr.write('%s\n' % (fileName))
valFilePtr.close()
display.self_print(
'\ttrain/val sizes: %d, %d' % (len(trainSet), len(valSet)),
'warning')
else:
display.self_print('\ttrain/val sizes: %d, 0' % (len(dataList)),
'warning')
trainFileOut = dataListDirs + os.path.sep + 'train.lst'
trainFilePtr = open(trainFileOut, 'w')
for fileName in dataList:
trainFilePtr.write('%s\n' % (fileName))
trainFilePtr.close()
# done
return
try:
leveltext = fodt_fmt['level'].format(level)
except:
leveltext = None
sums = []
tests = []
for spec in test_spec:
obj = getattr(thismodule, spec['type'])(*spec['ranges'])
tests.extend([
tst for tst in [obj.text(sums) for i in range(spec['count'])]
if tst is not None
])
# tests = sorted(tests)
if options.shuffle:
random_shuffle(tests)
if (len(tests) > 100):
print >> sys.stderr, fmt_warn_gt_100_tests.format(len(tests))
# print >> sys.stderr, (len(tests))
blank = Blank()
tests.extend([blank.text(sums) for x in range(len(tests), 100)])
ix = 0
with open(testfile, 'wb') as fp:
for line in lines:
sline, ix = substitute(line, tests, 'sum', ix)
fp.write(sline)
def shuffle(self):
random_shuffle(self._table)
def split_dataset(dataset, ratio=0.8, shuffle=False):
div = floor(len(dataset) * ratio)
if (shuffle):
random_shuffle(dataset)
return dataset[0:div], dataset[div:]
def fit(self,
X,
Y,
epochs,
batch_size,
Xval=None,
Yval=None,
shuffle=True,
iterations={
'training': None,
'validation': None
},
callbacks=None):
self.__testFreezeModel()
self.__testNetworkHasNotCompiled()
# Iniciamos los callbacks
if callbacks is None:
callbacks = [
PrettyMonitor(PrettyMonitor.TRAINING, 5),
PrettyMonitor(PrettyMonitor.VALIDATION)
]
for c in callbacks:
c.init(self)
# Redefinimos
total_epochs = epochs
enable_shuffle = shuffle
# Flags para los parametros
has_iterations = iterations is not None and iterations and isinstance(
iterations, dict)
enable_validation = Xval is not None and Yval is not None
# Definimos el numero de iteraciones maximas por el training
total_batchs = list(X.values())[0].shape[0]
if has_iterations and 'training' in iterations and \
iterations['training'] is not None:
total_batchs = min(total_batchs,
batch_size * iterations['training'])
total_iterations = math.ceil(total_batchs / batch_size)
# Definimos el numero de iteraciones maximas por el validation (si existe)
if enable_validation:
total_batchs_val = list(Xval.values())[0].shape[0]
if has_iterations and 'validation' in iterations and \
iterations['validation'] is not None:
total_batchs_val = min(total_batchs_val,
batch_size * iterations['validation'])
total_iterations_val = math.ceil(total_batchs_val / batch_size)
# Definimos los indices para realizar un shuffle de forma rapida
indices = list(range(list(X.values())[0].shape[0]))
if enable_validation:
indices_val = list(range(list(Xval.values())[0].shape[0]))
# Empezamos las epocas
for epoch in range(total_epochs):
# TRAINING
#
#
# Aplicamos el shuffle
if enable_shuffle:
random_shuffle(indices)
# Iniciamos la variables para training
batch_index = 0
iteration = 0
# Empezamos las iteraciones
for batch_index in range(0, total_batchs, batch_size):
# Definimos los intervalos del batch
start = batch_index
end = min(batch_index + batch_size, total_batchs)
# Cargamos el batch
X_batch = dict(
(k, v[indices[start:end]]) for k, v in X.items())
Y_batch = dict(
(k, v[indices[start:end]]) for k, v in Y.items())
# Ejecutamos callbacks
for c in callbacks:
c.excecute_pre_batch(self.SPLIT.TRAINING, iteration,
total_iterations, batch_index,
total_batchs, batch_size, epoch,
total_epochs)
# Entrenamos con ese batch
self.train_batch(X_batch, Y_batch)
# Ejecutamos callbacks
for c in callbacks:
c.excecute_post_batch(self.SPLIT.TRAINING, iteration,
total_iterations, batch_index,
total_batchs, batch_size, epoch,
total_epochs)
iteration += 1
# VALIDATION
#
#
# Comprobamos si hay que realizar validation
if enable_validation:
# Aplicamos el shuffle
if enable_shuffle:
random_shuffle(indices_val)
# Iniciamos la variables para training
batch_index = 0
iteration = 0
# Empezamos las iteraciones
for batch_index in range(0, total_batchs_val, batch_size):
# Definimos los intervalos del batch
start = batch_index
end = min(batch_index + batch_size, total_batchs)
# Cargamos el batch
X_batch = dict(
(k, v[indices[start:end]]) for k, v in X.items())
Y_batch = dict(
(k, v[indices[start:end]]) for k, v in Y.items())
# Ejecutamos callbacks
for c in callbacks:
c.excecute_pre_batch(self.SPLIT.VALIDATION, iteration,
total_iterations_val, batch_index,
total_batchs_val, batch_size,
epoch, total_epochs)
# Entrenamos con ese batch
self.__validate_batch(X_batch, Y_batch)
# Ejecutamos callbacks
for c in callbacks:
c.excecute_post_batch(self.SPLIT.VALIDATION, iteration,
total_iterations_val,
batch_index, total_batchs_val,
batch_size, epoch, total_epochs)
iteration += 1
