def generate_query_embeddings(model, instances_list):
batch_size = 100
embeddings_list = list()
print('Computing embeddings for %s questions' % len(instances_list))
slices = zip(*(iter(instances_list), ) * batch_size)
num_batches = int(len(instances_list) / batch_size)
for n, s in enumerate(slices):
output.clear(output_tags='progress')
with output.use_tags('progress'):
print('Processing batch %s of %s' % (n + 1, num_batches))
question_batch = list(
[question_dict["question"] for question_dict in s])
encodings = model.signatures['question_encoder'](
tf.constant(question_batch))
for i in range(len(question_batch)):
embeddings_list.append(np.array(encodings['outputs'][i]))
if num_batches * batch_size < len(instances_list):
question_batch = list([
question_dict["question"]
for question_dict in instances_list[num_batches * batch_size:]
])
encodings = model.signatures['question_encoder'](
tf.constant(question_batch))
for i in range(len(question_batch)):
embeddings_list.append(np.array(encodings['outputs'][i]))
return np.array(embeddings_list)
def GetCustomFiles(self, value):
output.clear()
self.from_to = (int(value[:value.find('-')]),int(value[value.find('-')+1:]))
print(f'Custom Vocab list created from word[{self.from_to[0]}] to word[{self.from_to[1]}]')
print('''
--------------------------------------------------------------------------------
Vocab file loaded! Please continue below!''')
def GetFiles(self, a):
output.clear()
self.choice = a
#if not os.path.isfile(a): os.popen(f'gsutil cp {file_dict[a]} .')
#assert os.path.isfile(a), 'Vocab file not found!'
self.string_ = self.GoogleBucket.get_blob(f'Vocab_txt/SAT-400/{a}').download_as_string().decode('ASCII')
if a != '999_vocab_SAT_ALL.txt':
print('''
--------------------------------------------------------------------------------
Vocab file loaded! Please continue below!''')
else:
print('Please input your custom word range:')
display(ReceiveInput('Input format: [start]-[end]', self.GetCustomFiles))
def render(self, mode='human'):
output.clear()
vertical_var = np.full((self.env_img.shape[0], 10, 3),
128,
dtype=np.uint8)
# cv2_imshow(data_img) #img_bgr[yv, xv])
print('\t\t Environment \t\t\t Drone View ')
cv2_imshow(np.hstack([self.env_img, vertical_var, self.drone_map]))
# print('\t\t Coverage \t\t\t Obstacles ')
# cv2_imshow(np.hstack([self.coverage, 255*self.obstacle_map]))
print('\t\t Coverage \t\t\t Drone Coverage ')
cv2_imshow(
np.hstack([
self.coverage, vertical_var[:, :, 0], self.drone_map[:, :, 1]
]))
def generate_document_embeddings(model, response_list, sent_list, doc_list):
'''
:param model: the use-QA model
:param response_list: a tuple (sent_id, doc_id)
:param sent_list: a list of strings
:param doc_list: a list of strings
:return:
'''
batch_size = 100
embeddings_list = list()
print('Computing embeddings for %s sentences' % len(response_list))
slices = zip(*(iter(response_list), ) * batch_size)
num_batches = int(len(response_list) / batch_size)
for n, s in enumerate(slices):
output.clear(output_tags='progress')
with output.use_tags('progress'):
print('Processing batch %s of %s' % (n + 1, num_batches))
response_batch = list(
[sent_list[int(sent_id)] for sent_id, doc_id in s])
context_batch = list([doc_list[int(doc_id)] for sent_id, doc_id in s])
encodings = model.signatures['response_encoder'](
input=tf.constant(response_batch),
context=tf.constant(context_batch))
for i in range(len(response_batch)):
embeddings_list.append(np.array(encodings['outputs'][i]))
if batch_size * num_batches < len(response_list):
response_batch = list([
sent_list[int(sent_id)]
for sent_id, doc_id in response_list[num_batches * batch_size:]
])
context_batch = list([
doc_list[int(doc_id)]
for sent_id, doc_id in response_list[num_batches * batch_size:]
])
encodings = model.signatures['response_encoder'](
input=tf.constant(response_batch),
context=tf.constant(context_batch))
for i in range(len(response_batch)):
embeddings_list.append(np.array(encodings['outputs'][i]))
return np.array(embeddings_list)
def fit(self):
for epoch in range(self.epoch, self.config.SOLVER.MAX_EPOCHS):
if epoch < self.config.SOLVER.WARMUP_EPOCHS:
lr_scale = min(
1.,
float(epoch + 1) / float(self.config.SOLVER.WARMUP_EPOCHS))
for pg in self.optimizer.param_groups:
pg['lr'] = lr_scale * self.config.SOLVER.BASE_LR
self.do_scheduler = False
else:
self.do_scheduler = True
if self.config.VERBOSE:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.logger.info(f'\n{timestamp}\nLR: {lr}')
t = time.time()
summary_loss = self.train_one_epoch()
self.logger.info(
f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}'
)
self.save(f'{self.base_dir}/last-checkpoint.bin')
t = time.time()
best_score_threshold, best_final_score = self.validation()
self.logger.info(
f'[RESULT]: Val. Epoch: {self.epoch}, Best Score Threshold: {best_score_threshold:.2f}, Best Score: {best_final_score:.5f}, time: {(time.time() - t):.5f}'
)
if best_final_score > self.best_final_score:
self.best_final_score = best_final_score
self.best_score_threshold = best_score_threshold
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint.bin')
self.save_model(f'{self.base_dir}/best-model.bin')
self.save_predictions(f'{self.base_dir}/all_predictions.csv')
self.early_stop(best_final_score)
if self.early_stop_epochs > self.config.SOLVER.EARLY_STOP_PATIENCE:
self.logger.info('Early Stopping!')
break
if self.epoch % self.config.SOLVER.CLEAR_OUTPUT == 0:
output.clear()
self.epoch += 1
def render(self, env):
self.screen.fill(WHITE)
self.render_robot(env.robot)
self.render_walls(env.walls)
self.render_pois(env.pois)
self.render_goal(env.goal)
#self.render_aoi()
#pygame.display.update()
if colab_use:
#convert image so it can be displayed in OpenCV
view = pygame.surfarray.array3d(self.screen)
# convert from (width, height, channel) to (height, width, channel)
view = view.transpose([1, 0, 2])
# convert from rgb to bgr
img_bgr = cv2.cvtColor(view, cv2.COLOR_RGB2BGR)
#Display image, clear cell every 0.5 seconds
cv2_imshow(img_bgr)
time.sleep(0.5)
output.clear()
def _clear_component(self, component_id=None, wait=False):
"""Clears component.
If component_id is None, it will clear currently active component,
otherwise it will clear one with given id.
NOTE FOR SUBCLASS IMPLEMENTTERS:
When _clear_output is called it will remove all outputs that were created
within context of _active_component.
This might produce subtle errors in situations where user clears component
he is currently producing output for as it will destroy any output that
is in the context of _active_component. Therefore if your widget
needs javascript to setup the component for output it should always
be produced by overloading _prepare_component_for_output.
Args:
component_id: which component to clear.
wait: if True, the output won't be cleared until the next user output.
See colab.output.clear for full details.
Raises:
WidgetException: if component_id and no active element is
selected
"""
_util.flush_all()
if component_id is None:
if self._current_component is None:
raise WidgetException('No active component selected')
component_id = self._current_component
if component_id == self._current_component:
# Do not clear the part that sets current active element.
# If we did, this would have made all consecutive output to stream
# to wrong outputarea on reload.
output.clear(wait, output_tags=[component_id] + ['user_output'])
else:
output.clear(wait, output_tags=[component_id])
def generate_document_embeddings_no_context(model, response_list):
'''
:param model: the use-QA model
:param response_list: a list of strings
:return:
'''
batch_size = 100
embeddings_list = list()
print('Computing embeddings for %s sentences' % len(response_list))
slices = zip(*(iter(response_list), ) * batch_size)
num_batches = int(len(response_list) / batch_size)
for n, s in enumerate(slices):
output.clear(output_tags='progress')
with output.use_tags('progress'):
print('Processing batch %s of %s' % (n + 1, num_batches))
# according to https://tfhub.dev/google/universal-sentence-encoder-qa/3, we should repeat answer_batch if there is no context.
answer_batch = list([sent for sent in s])
#context_batch = list([" "])
encodings = model.signatures['response_encoder'](
input=tf.constant(answer_batch), context=tf.constant(answer_batch))
for i in range(len(answer_batch)):
embeddings_list.append(np.array(encodings['outputs'][i]))
if batch_size * num_batches < len(response_list):
answer_batch = list(
[sent for sent in response_list[num_batches * batch_size:]])
encodings = model.signatures['response_encoder'](
input=tf.constant(answer_batch), context=tf.constant(answer_batch))
for i in range(len(answer_batch)):
embeddings_list.append(np.array(encodings['outputs'][i]))
return np.array(embeddings_list)
def _clear_component(self, component_id=None, wait=False):
"""Clears currently active component.
Args:
component_id: which component to clear.
wait: if True, the output won't be cleared until the next user output.
See colab.output.clear for full details.
Raises:
WidgetException: if component_id and no active element is
selected
"""
_util.flush_all()
if component_id is None:
if self._current_component is None:
raise WidgetException('No active component selected')
component_id = self._current_component
if component_id == self._current_component:
# Do not clear the part that sets current active element.
# If we did, this would have made all consecutive output to stream
# to wrong outputarea on reload.
output.clear(wait, output_tags=[component_id] + ['user_output'])
else:
output.clear(wait, output_tags=[component_id])
def run_predictions(eval_set, out_filename, n_examples=None, padding=False):
"""
Automatic prediction script with options for checkpointing, error coverage,
padding of incomplete experiments and filedumping.
"""
predictions = dict()
checkpoints = []
error_count = 0
for k in range(1000, 11000, 1000):
# checkpoints.append(k)
pass
if not n_examples:
for i, example in enumerate(eval_set):
try:
predictions[example.qas_id] = answer_question(example.question_text, example.context_text)
except:
predictions[example.qas_id] = ""
error_count += 1
output.clear('status_text')
with output.use_tags('status_text'):
print(f"Loaded {i+1}/{len(eval_set)} Failed predictions: {error_count}")
if i in checkpoints:
with open('/content/gdrive/My Drive/squad/'+str(int(i/1000))+'k_'+out_filename, 'w') as fp:
json.dump(predictions, fp)
print(f"Saved checkpoint {i/1000}k")
else:
for i in range(n_examples):
example = eval_set[i]
predictions[example.qas_id] = answer_question(example.question_text, example.context_text)
output.clear('status_text')
with output.use_tags('status_text'):
print(f"Loaded {i+1}/{n_examples}")
if padding:
for i in range(n_examples, len(eval_set), 1):
example = eval_set[i]
predictions[example.qas_id] = ""
output.clear('status_text')
with output.use_tags('status_text'):
print(f"Loaded {i+1}/{len(eval_set)}")
output.clear('status_text')
with open('/content/gdrive/My Drive/squad/'+out_filename, 'w') as fp:
json.dump(predictions, fp)
def setup(
ngrok_region=None,
check_gpu_available=True,
tunnel="ngrok",
public_key=None,
ngrok_key=None,
secret_key=None,
vncserver=False,
):
print("[!] Setup process started")
stat, msg = _setupSSHDMain(
public_key,
tunnel,
ngrok_region,
check_gpu_available,
True,
ngrok_key,
secret_key,
)
if stat:
if vncserver:
_setupVNC(secret_key)
output.clear()
print(msg)
Original file is located at
https://colab.research.google.com/drive/1nQcQVYyrcPsd6q_qW3WUAkpv618oIuc4
##**Gender-speech-duration-calculator**
Here you will find a tool to calculate the percentage of time of female voice speech and male voice speech in a video/movie. You can either choose to paste a youtube link or you can upload your video. Install the program, choose one option and calculate the percentage of female and male speech.
##**Step 1: installation**
"""
#@title Install gender-speech-duration-calculator
!apt-get install ffmpeg
!pip install inaSpeechSegmenter
!pip install youtube_dl
!pip install -qU ddsp
from google.colab import output
output.clear()
"""##**Step 2: upload/paste video**"""
#@title Option 1: paste your youtube link here
import youtube_dl
youtube_link= 'https://www.youtube.com/watch?v=UG_X_7g63rY&t=23s' #@param {type:"string"}
ydl_opts = {
'format': 'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
'outtmpl': 'audio.%(etx)s',
'quiet': False
if not wrote_to_fifo:
with open(fifo, 'w') as fifo_file:
fifo_file.write('ignored\n')
filtered_logfile = _timeouts_path()
d.sendline('fuser -kw "{f}" ; rm -rf "{f}"'.format(f=filtered_logfile))
d.expect(prompt)
filter_script = _os.path.join(drive_dir, 'drive-filter.py')
filter_cmd = (
"""nohup bash -c 'tail -n +0 -F "{}" | """
"""python3 {} > "{}" ' < /dev/null > /dev/null 2>&1 &""").format(
dfs_log, filter_script, filtered_logfile)
d.sendline(filter_cmd)
d.expect(prompt)
if 'ENABLE_DIRECTORYPREFETCHER' in _os.environ:
d.sendline(
"""nohup bash -c '{d}/directoryprefetcher_binary -mountpoint={mnt}' """
""">> {log} 2>&1 &""".format(d=drive_dir,
mnt=mountpoint,
log=_os.path.join(
_logs_dir(), 'dpb.txt')))
d.expect(prompt)
d.sendline('disown -a')
d.expect(prompt)
d.sendline('exit')
assert d.wait() == 0
_output.clear(wait=True, output_tags='dfs-auth-dance')
print('Mounted at {}'.format(mountpoint))
mount._DEBUG = False # pylint:disable=protected-access
def main():
window_size = 6
batch_size = 16
model_name = 'skipgram-win12'
ckpt_dir = '/content/gdrive/My Drive/files/{0}/{0}'.format(model_name)
ckpt_dir = ckpt_dir + '-ep{}.ckpt'
text, dictionary = util.nltk2data(brown, save_dict=False, remove_punc=True)
train_inps, train_labels, eval_inps, eval_labels = util.data_dicer(
text,
dictionary,
window_size,
batch_size,
chopoff=True,
train_eval_split=0.8)
# TODO: placeholders for input and output (label)
vocab_size = len(dictionary.keys())
embed_dim = 512
num_sampled = 10
num_true = 2 * window_size
epochs = 1
current_epoch = 1
x_ph = tf.placeholder(tf.int32, shape=(None, ), name='x_ph')
target_ph = tf.placeholder(tf.int32,
shape=(None, num_true),
name='target_ph')
# TODO: construct embedding_layers USING VARIABLE SCOPE!
with tf.variable_scope('skipgram', reuse=tf.AUTO_REUSE):
embedding_layer = tf.get_variable('embedding_layer',
shape=(vocab_size, embed_dim),
dtype=tf.float32,
initializer=tf.constant_initializer(
np.random.randn(
vocab_size, embed_dim)))
output_weights = tf.get_variable('output_weight',
shape=(vocab_size, embed_dim),
dtype=tf.float32,
initializer=tf.constant_initializer(
np.random.randn(
vocab_size, embed_dim)))
output_biases = tf.get_variable('output_bias',
shape=(vocab_size, ),
dtype=tf.float32,
initializer=tf.zeros_initializer())
# TODO: map the word using tf.nn.lookup
# center_word shape = [batch_size, embed_dim]
center_word = tf.nn.embedding_lookup(embedding_layer, x_ph)
# TODO: Check which mode, either training or eval
# TODO: Training, calculate NCE loss
train_loss = tf.nn.nce_loss(weights=output_weights,
biases=output_biases,
labels=target_ph,
inputs=center_word,
num_true=num_true,
num_sampled=num_sampled,
num_classes=vocab_size)
train_batch_loss = tf.reduce_mean(train_loss)
opt = tf.train.AdamOptimizer().minimize(train_batch_loss)
# TODO: eval, calculate manually? refers to tensorflow guide.
# output_weights.shape = [vocab_size, dim]
# center_word.shape = [batch_size, dim]
# matmul, output_weights
# projections = tf.matmul(center_word, tf.transpose(output_weights)) + output_biases
# eval_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=projections, labels=tf.squeeze(target_ph))
# eval_batch_loss = tf.reduce_mean(eval_loss)
saver = tf.train.Saver(tf.trainable_variables())
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
if current_epoch > 1:
saver.restore(sess, ckpt_dir.format(current_epoch - 1))
start_time = time.time()
print("=" * 80)
print("=" * 80)
print("Start time: {}".format(
time.strftime('%Y/%m/%d, %H:%M%S', time.localtime(start_time))))
print("=" * 80)
print("=" * 80)
for i in range(epochs):
epoch_start_time = time.time()
print("Epoch: ", i + 1)
epoch_loss = []
# Do it over batches
# 1. stats
batch_number = 0
seconds_per_batch = 0
total_batch = len(train_inps)
for batch_inps, batch_labels in zip(train_inps, train_labels):
batch_start_time = time.time()
batch_number = batch_number + 1
left_over_time = round(
(total_batch + 1 - batch_number) * seconds_per_batch, 3)
output.clear(output_tags=('batch_print'), wait=True)
with output.use_tags('batch_print'):
print(
"Current batch:{}/{}\tSeconds per batch: {}s, {}s left."
.format(batch_number, total_batch + 1,
seconds_per_batch, left_over_time))
feed_dict = {x_ph: batch_inps, target_ph: batch_labels}
_, batch_loss_v = sess.run([opt, train_batch_loss],
feed_dict=feed_dict)
epoch_loss.append(batch_loss_v)
batch_end_time = time.time()
seconds_per_batch = round(batch_end_time - batch_start_time, 3)
# TODO: averaging epoch_loss
feed_dict = {x_ph: eval_inps, target_ph: eval_labels}
[eval_loss_v] = sess.run([train_batch_loss], feed_dict=feed_dict)
epoch_loss = np.mean(epoch_loss)
# TODO: print train loss, print eval loss
epoch_end_time = time.time()
epoch_duration = epoch_end_time - epoch_start_time
print("Epochs took: {}s".format(round(epoch_duration, 3)))
print("Train Loss: {}\tEval Loss: {}".format(
round(epoch_loss, 3), round(eval_loss_v, 3)))
end_time = time.time()
duration = end_time - start_time
print("=" * 80)
print("=" * 80)
print("End time: {}".format(
time.strftime('%Y/%m/%d, %H:%M%S', time.localtime(end_time))))
print("Duration: {}s".format(round(duration, 2)))
print("=" * 80)
print("=" * 80)
saver.save(sess, ckpt_dir.format(current_epoch))
def main(data_dir, query_condition, ex_list=None):
dem_dir = os.path.join(data_dir, "dem")
dembz_dir = os.path.join(data_dir, "dembz")
log_dir = os.path.join(data_dir, "log")
csv_dir = os.path.join(data_dir, "csv")
last_match_id = ""
if ex_list is None:
exist_list = [fn.name.split("_")[0] for fn in list(Path(csv_dir).glob("*.csv"))]
else:
exist_list = ex_list
while True:
matches_json = get_match_id(last_match_id)
if len(matches_json)<=0:
last_match_id = ""
continue
df_matches_all = pd.DataFrame(matches_json)
last_match_id = df_matches_all['match_id'].values[-1]
# matches_file = os.path.join(data_dir, "data_5236503318_8w.csv")
# df_matches_all = pd.read_csv(matches_file)[20000:]
df_matches = df_matches_all.query(query_condition).reset_index()
from google.colab import output
output.clear()
for i in range(df_matches.shape[0]):
print(i, "/",df_matches.shape[0])
match_id = df_matches['match_id'][i]
replay_json = get_replay_salt(match_id)
cluster = replay_json[0]["cluster"]
replay_salt = replay_json[0]["replay_salt"]
done, filename = download_replay(match_id, cluster, replay_salt)
if done:
try:
t = time.time()
os.system('curl localhost:5600 --data-binary "@%s.dem" > %s.json | exit 1'%(filename, filename))
os.system("python ParseJsonRecord.py %s.json %s.csv"%(filename, filename))
dur = time.time()-t
print("P", dur)
df_matches.loc[i].to_json("%s.info"%filename)
assert os.path.exists(filename+".dem.bz2")
assert os.path.exists(filename+".dem")
assert os.path.exists(filename+".json")
assert os.path.exists(filename+".csv")
assert os.path.exists(filename+".info")
# shutil.move(filename+".dem.bz2", os.path.join(dembz_dir, filename+".dem.bz2"))
# shutil.move(filename+".dem", os.path.join(dem_dir, filename+".dem"))
# shutil.move(filename+".json", os.path.join(log_dir, filename+".json"))
os.remove(filename+".dem.bz2")
os.remove(filename+".dem")
os.remove(filename+".json")
shutil.move(filename+".info", os.path.join(csv_dir, filename+".info"))
shutil.move(filename+".csv", os.path.join(csv_dir, filename+".csv"))
exist_list.append(match_id)
print("saved", filename)
except:
exist_list.append(match_id)
print("save failed", filename)
raise
val = random.random()
if val > 1 - SKIP_PERCANTAGE:
continue
# Get the tokens and remove words that are too short
tokens = []
try:
tokens = row['cleaned_Data_Content'].split(' ')
tokens = [token for token in tokens if len(token) > 3]
except:
print('A book failed')
books.append(tokens)
genres.append(genre)
output.clear('status_text')
with output.use_tags('status_text'):
print('Books loaded: ' + str(count))
count += 1
genres = np.array(genres)
books = np.array(books)
# Additional data processing to optimize the topics generated
commonWords = []
with open("/content/drive/My Drive/ATIML/common.txt") as common:
for word in common:
commonWords.append(word[:-1])
def train(self, fine_tuning = False):
self._check_required_models()
start_train_wall_time = datetime.datetime.now()
for callback in self._tensorboard_callback_dict.values():
callback.on_train_begin()
gpus = tf.config.experimental.list_physical_devices('GPU')
n_gpus = len(gpus)
if self._verbose: print('This machine has %i GPUs.' % n_gpus)
# containers for losses
lc = Container(); self.lc = lc
if self._load_model_at_path is not None:
# Load model
self.load( self._load_model_at_path )
# Load loss record
self._fill_training_container( self._load_model_at_path, lc )
if self._load_model_at_path != self._save_model_at_path:
# Save a model copy with previous best validation so that if this
# training session is useless, we can recover previous best model
self.save( overwrite = True, val = True )
else:
lc.surrogate_loss_record = {k : [] for k in self._surrogate_lkeys}
lc.train_perf_record = {}; lc.val_perf_record = {}
lc.best_epoch = lc.best_step = lc.last_progress_step = 0; lc.best_val_reco = np.finfo( dtype = np.float32 ).max
lc.p_sample_period = -np.inf; lc._history_cur_batch_samples = 0;
lc.step = lc.epoch = 0
lc.prev_train_time = datetime.timedelta()
lc.total_performance_measure_time = datetime.timedelta()
last_print_cycle = -1; last_save_cycle = 0
skipFinalPerfEval = is_new_print_cycle = False;
exc_type = exc_val = None
# When fine tuning, we need to reset the validation dataset statistics
if fine_tuning:
lc.best_val_reco = np.finfo( dtype = np.float32 ).max
train_perf_dict = {}
val_perf_dict = {}
lc.first_step = True
lc.session_performance_measure_time = datetime.timedelta()
lc.last_performance_measure_time = datetime.timedelta()
lc.first_step_measure_time = datetime.timedelta()
lc.session_step = 0
lc.session_epoch = 0
n_measurements = 0
try:
while (lc.epoch < self._max_epoches if self._max_epoches else True):
alreadyPrintedEpoch = alreadySavedEpoch = False
for sample_batch in self.data_sampler.training_sampler:
with DelayedKeyboardInterrupt():
if self._log_models_in_tensorboard:
for callback in self._tensorboard_callback_dict.values():
callback.on_epoch_begin(epoch = lc.step)
callback.on_train_batch_begin(batch = lc.step)
evaluatedPerf = False
# TODO To measure performance on purely initialize sample, simply run
# below without running self._train_base and without incrementing
# lc.step
if self.sample_parser_fcn is not None:
#self.data_sampler.plot(data_samples = sample_batch, do_display = True)
sample_batch = self.sample_parser_fcn(sample_batch)
#print("Running first train step")
surrogate_loss_dict = self._train_base(lc.epoch, lc.step, sample_batch)
#print("Finished computing and updating one train step")
lc.step += 1; lc.session_step += 1
start_performance_measure = datetime.datetime.now()
surrogate_loss_dict = self._parse_surrogate_loss( surrogate_loss_dict )
surrogate_loss_dict['step'] = lc.step
# Keep track of training record:
# TODO This should be integrated in the meters, i.e. compute loss only if passing constraint below
c_sample_period = np.log10( lc.step ) // self._log_sampling_period
if c_sample_period > lc.p_sample_period:
lc.p_sample_period = c_sample_period
lc._history_cur_batch_samples = 0
if lc._history_cur_batch_samples < self._history_max_batch_samples:
# NOTE handle_new_loss_step keeps track of what is plot/logged
# during training
# TODO This should be integrated with the meter framework
#print("Keeping track of surrogate loss")
with self._surrogate_summary_writer.as_default(step = lc.step) as writer:
self._handle_new_loss_step( lc.surrogate_loss_record, surrogate_loss_dict
, keys = self._surrogate_lkeys )
for fcn in self._other_surrogate_logging_fcns:
fcn(surrogate_loss_dict)
# Compute efficiency
if ( not(lc.step % self._n_performance_measure_steps) or lc.step == 1):
n_measurements += 1
#print("Computing train dataset performance")
#if lc._history_cur_batch_samples < self._history_max_batch_samples:
train_perf_dict = self.performance_measure_fcn(
sampler_ds = self.data_sampler.evaluation_sampler_from_train_ds,
meters = self._train_perf_meters)
train_perf_dict['step'] = lc.step
self.lc.last_train_perf_step = lc.step
with self._train_perf_summary_writer.as_default(step = lc.step) as writer:
self._handle_new_loss_step(lc.train_perf_record, train_perf_dict)
for fcn in self._other_train_perf_logging_fcns:
fcn(train_perf_dict)
# Compute performance for validation dataset (when available)
if self.data_sampler.has_val_ds:
#print("Computing val dataset performance")
val_perf_dict = self.performance_measure_fcn(
sampler_ds = self.data_sampler.evaluation_sampler_from_val_ds,
meters = self._val_perf_meters )
val_perf_dict['step'] = lc.step
self.lc.last_val_perf_step = lc.step
evaluatedPerf = True
with self._val_perf_summary_writer.as_default(step = lc.step) as writer:
# TODO Reminder handle new loss keeps track of what is plot/logged
# during training. Should be integrated with meter framework
self._handle_new_loss_step(lc.val_perf_record, val_perf_dict )
for fcn in self._other_val_perf_logging_fcns:
fcn(val_perf_dict)
# Early stopping algo: Keep track of best model so far
#print("Computing early stopping")
if bool(self.early_stopping_key) and val_perf_dict[self.early_stopping_key] < lc.best_val_reco:
if lc.best_val_reco - val_perf_dict[self.early_stopping_key] > self._min_progress:
lc.last_progress_step = lc.step
lc.best_val_reco = val_perf_dict[self.early_stopping_key]
lc.best_step = lc.step; lc.best_epoch = lc.epoch
self.save( overwrite = True, val = True )
# Check whether to break due to
if ( lc.step - lc.last_progress_step ) >= self._max_fail:
raise BreakDueToMaxFail()
# End of efficiency computation
# Performed one model update step
# Compute training time
lc.session_train_time = datetime.datetime.now() - start_train_wall_time
print_cycle = int( lc.session_train_time / self._print_interval_wall_time ) if self._print_interval_wall_time is not None else 0
is_new_print_cycle = print_cycle > last_print_cycle
# Compute performance measurement time:
stop_performance_measure = datetime.datetime.now()
this_step_performance_measure_time = stop_performance_measure - start_performance_measure
if evaluatedPerf:
lc.total_performance_measure_time += this_step_performance_measure_time
lc.session_performance_measure_time += this_step_performance_measure_time
lc.last_performance_measure_time = this_step_performance_measure_time
if lc.first_step:
lc.first_step_measure_time = this_step_performance_measure_time
# Print/plot loss
if ((self._verbose or self._online_train_plot) and
(
(not(lc.step % self._print_interval_steps) if self._print_interval_steps is not None else False)
or ((not(lc.epoch % self._print_interval_epoches) if self._print_interval_epoches is not None else False) and not(alreadyPrintedEpoch) )
or is_new_print_cycle
)
):
#print("Proceeding to printing")
last_improvement = { 'best_val_reco' : lc.best_val_reco
, 'best_step' : lc.best_step
, 'last_progress_step' : lc.last_progress_step } if val_perf_dict else {}
if self._online_train_plot:
try:
from google.colab import output
output.clear()
except ImportError:
from IPython.display import clear_output
clear_output(wait = True)
plt.close('all')
self.plot_surrogate_progress( do_display = True )
self.plot_performance_progress( do_display = True )
for fcn in self._online_plot_fcns:
fcn()
if self._verbose:
self._replace_nans_with_last_report( surrogate_loss_dict, lc.surrogate_loss_record )
self._print_progress( lc.epoch, lc.session_epoch, lc.step, lc.session_step
, lc.prev_train_time, lc.session_train_time, lc.session_performance_measure_time
, lc.last_performance_measure_time
, lc.first_step_measure_time
, n_measurements
, surrogate_loss_dict, train_perf_dict, val_perf_dict, last_improvement )
if not(lc.epoch % self._print_interval_epoches) if self._print_interval_epoches is not None else False:
alreadyPrintedEpoch = True
if is_new_print_cycle:
last_print_cycle = print_cycle
is_new_print_cycle = False
# Finished printing
# Check whether we have finished training
if self._max_steps is not None and (lc.step > self._max_steps):
raise BreakDueToUpdates()
if self._max_train_wall_time is not None and (lc.session_train_time > self._max_train_wall_time):
raise BreakDueToWallTime()
# No halt requested. Increament counters
if lc._history_cur_batch_samples < self._history_max_batch_samples:
lc._history_cur_batch_samples += 1
save_cycle = int( lc.session_train_time / self._save_interval_wall_time ) if self._save_interval_wall_time is not None else 0
is_new_save_cycle = save_cycle > last_save_cycle
# Save progress. Note that this save is not due to early stopping,
# but rather to allow recovering current weights regardless of
# training status
if (
(not(lc.step % self._save_interval_steps) if self._save_interval_steps is not None else False)
or ((not(lc.epoch % self._save_interval_epoches) if self._save_interval_epoches is not None else False) and not(alreadySavedEpoch) )
or is_new_save_cycle
):
#print("Saving progress")
loss_data = { 'surrogate_loss_record' : lc.surrogate_loss_record
, 'train_perf_record' : lc.train_perf_record
, 'val_perf_record' : lc.val_perf_record }
self.save( overwrite = True
, loss_data = loss_data
, locals_data = lc )
if not(lc.epoch % self._save_interval_epoches) if self._save_interval_epoches is not None else False:
alreadySavedEpoch = True
if is_new_save_cycle:
last_save_cycle = save_cycle
is_new_save_cycle = False
if self._log_models_in_tensorboard:
for callback in self._tensorboard_callback_dict.values():
callback.on_epoch_end(epoch = lc.step)
callback.on_train_batch_end(batch = lc.step, logs = val_perf_dict)
if ( not(lc.step % self._n_performance_measure_steps) or lc.step == 1):
self._update_writer_file(self._surrogate_summary_writer)
self._update_writer_file(self._train_perf_summary_writer)
self._update_writer_file(self._val_perf_summary_writer)
lc.first_step = False
# end of step, send delayed keyboard interrupt and allow interruptions
lc.epoch += 1; lc.session_epoch += 1
# Performed a full pass through training dataset
raise BreakDueToEpoches
except BaseException as e:
exc_type, exc_val = sys.exc_info()[:2]
finally:
if isinstance( exc_val, InterruptTraining):
print('Training finished!')
interruptTraining = True
if isinstance( exc_val, BreakDueToMaxFail ):
# Recover best validation result
print('Reason: early stopping.')
print('Recovering Best Validation Performance @ (Epoch %i, Step %i).' % (lc.best_epoch, lc.best_step,))
print('Reco_loss: %.3f.' % lc.best_val_reco)
self.load( self._save_model_at_path, val = True )
skipFinalPerfEval = True
elif isinstance( exc_val, BreakDueToUpdates ):
print('Reason: max steps.')
elif isinstance( exc_val, BreakDueToEpoches ):
print('Reason: max epoches.')
elif isinstance( exc_val, BreakDueToWallTime):
print('Reason: reached wall time limit.')
# Other non-critical interruptions
elif isinstance( exc_val, KeyboardInterrupt):
interruptTraining = True
print('Training finished!')
print('Reason: user interrupted training.')
# Critical interruptions
elif isinstance( exc_val, (TrainingCriticalAbort, BaseException) ):
print('ERROR: Training aborted!')
if isinstance( exc_val, BreakDueToNonFinite ):
print('Reason: found non-finite value!!')
raise exc_val
if self.data_sampler.has_val_ds:
if not skipFinalPerfEval:
if not evaluatedPerf:
with self._train_perf_summary_writer.as_default(step = lc.step) as writer:
train_perf_dict = self.performance_measure_fcn(
sampler_ds = self.data_sampler.evaluation_sampler_from_train_ds,
meters = self._train_perf_meters
)
train_perf_dict['step'] = lc.step
self._handle_new_loss_step(lc.train_perf_record, train_perf_dict )
self._update_writer_file(self._train_perf_summary_writer)
with self._val_perf_summary_writer.as_default(step = lc.step) as writer:
val_perf_dict = self.performance_measure_fcn(
sampler_ds = self.data_sampler.evaluation_sampler_from_val_ds,
meters = self._val_perf_meters
)
val_perf_dict['step'] = lc.step
self._handle_new_loss_step(lc.val_perf_record, val_perf_dict )
self._update_writer_file(self._val_perf_summary_writer)
if bool(self.early_stopping_key):
if ( lc.step == lc.best_step or val_perf_dict[self.early_stopping_key] < lc.best_val_reco ):
lc.best_val_reco = val_perf_dict[self.early_stopping_key]
lc.best_step = lc.step; lc.best_epoch = lc.epoch
else:
print('Validation Performance @ (Epoch %i, Step %i): %f.' % (lc.epoch, lc.step, val_perf_dict[self.early_stopping_key]))
print('Recovering Best Validation Performance @ (Epoch %i, Step %i).' % (lc.best_epoch, lc.best_step,))
print('Reco_loss: %.3f.' % (lc.best_val_reco))
self.load( self._save_model_at_path, val = True )
self.save( overwrite = True, locals_data = lc )
for callback in self._tensorboard_callback_dict.values():
callback.on_train_end()
# Compute final performance:
final_performance = {}
final_performance['train'] = self.performance_measure_fcn(
sampler_ds = self.data_sampler.evaluation_sampler_from_train_ds,
meters = self._train_perf_meters )
if self.data_sampler.has_val_ds:
final_performance['val'] = self.performance_measure_fcn(
sampler_ds = self.data_sampler.evaluation_sampler_from_val_ds,
meters = self._val_perf_meters )
final_performance['val']['best_step'] = lc.best_step
final_performance['val']['best_epoch'] = lc.best_epoch
else:
final_performance['val'] = dict()
loss_data = { 'surrogate_loss_record' : lc.surrogate_loss_record
, 'train_perf_record' : lc.train_perf_record
, 'val_perf_record' : lc.val_perf_record
, 'final_performance' : final_performance }
self.save( save_models_and_optimizers = False
, loss_data = loss_data)
return loss_data
IS_TPU = False
if os.environ.get('IS_COLAB', False):
from google.colab import output
try:
with output.use_tags('setup'):
# !pip install convokit
# !python3 -m spacy download en
tpu = tf.distribute.cluster_resolver.TPUClusterResolver(
) # TPU detection
print('Running ovariable_namen TPU ',
tpu.cluster_spec().as_dict()['worker'])
tf.config.experimental_connect_to_cluster(tpu)
tf.tpu.experimental.initialize_tpu_system(tpu)
tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)
IS_TPU = True
output.clear(output_tags='setup')
except ValueError:
logging.info('Not connected to a TPU runtime')
def make_model(tokenizer=None,
num_layers=2,
units=512,
d_model=256,
num_heads=8,
dropout=0.1,
max_length=32,
warmup_steps=4000):
logging.info('Compiling model.')
var iddoc=document.getElementById('colorized');
iddoc.style.color=d3.interpolate"""+cmap+"""((now%60000)/60000);},1)
</script>"""
display(HTML(html_str))
# %cmap_header CODE MODULES & HELPFUL TOOLS|24|Ewert|Turbo
"""#✒️ Tagged Outputs"""
import sys,time
print('The process is starting')
with output.use_tags('tagged_outputs'):
for el in ['working \n','=> => => \n','still working \n']:
sys.stdout.write('working \n')
sys.stdout.flush(); time.sleep(5)
output.clear(output_tags='tagged_outputs')
print('Outputs have cleared')
"""#✒️ Linked Outputs"""
# Commented out IPython magic to ensure Python compatibility.
# %%javascript
# const listener=new BroadcastChannel('channel1');
# listener.onmessage=(msg)=>{
# const div=document.createElement('div');
# div.textContent=msg.data;
# div.style.border='double white';
# div.style.width='20%';
# div.style.padding='20px';
# document.body.appendChild(div); };
