def get_pairs(): """ Create a list with tuples with pairs of airports. Like: [("BCN", "CAG"), ("GRO", "CAG")] This is created using all possible combinations from: congig.cfg/AIRPORTS/ORIGINDS And might be limited the size using congig.cfg/AIRPORTS/LIMIT """ # My airports airports = config["AIRPORTS"]["ORIGINS"].split(",") pairs = [] for origin in airports: for dest in airports: if dest != origin: # Append flights from both directions pairs.append((dest, origin)) pairs.append((origin, dest)) log.info(f"There are {len(pairs)} airport pairs") limit = int(config["AIRPORTS"]["LIMIT"]) if limit > 0: pairs = pairs[:limit] log.info(f"Limiting the query to {limit} pairs") return pairs
def eval_step(input_ids, target_ids, ): target_inp = target_ids[:, :-1] _, combined_mask, dec_padding_mask = create_masks(input_ids, target_inp) (draft_predictions, draft_attention_weights, refine_predictions, refine_attention_weights) = Model( input_ids, dec_padding_mask=dec_padding_mask, target_ids=target_inp, look_ahead_mask=combined_mask, training=False ) loss, target = loss_function(target_ids, draft_predictions, refine_predictions, Model ) train_loss(loss) log.info(Model.summary()) if config.save_initial_weights: initial_weights = os.path.join(config.initial_weights,'initial_weights') Model.save_weights(initial_weights) return loss
def save_evaluate_monitor(ck_pt_mgr, val_dataset, target_tokenizer, predictions, target_ids, step, start_time, bert_f1_score): ckpt_save_path = ck_pt_mgr.save() # print the detokenized training output of a single sample train_sanity_check(target_tokenizer, predictions, target_ids, log) # Run evaluation only if the train_loss is lesser than validate_when_train_loss_is if train_loss.result() < config.validate_when_train_loss_is: (validation_perplexity, val_bert_score, draft_attention_weights, refine_attention_weights) = evaluate_validation_set(val_dataset) early_stop_training = monitor_eval_metrics(ckpt_save_path, validation_perplexity, val_bert_score, train_loss.result(), step, log, config) else: log.info( 'Not running evaluation since loss is not lesser than config.validate_when_train_loss_is' ) (validation_perplexity, val_bert_score) = (0, 0) early_stop_training = False draft_attention_weights = None refine_attention_weights = None training_results(step, train_loss.result(), bert_f1_score.numpy(), validation_perplexity, val_bert_score, (time.time() - start_time), ckpt_save_path, log, config) train_loss.reset_states() return (early_stop_training, draft_attention_weights, refine_attention_weights)
def main(): """ Read the airport data from internet and store it as a pickle """ df = get_airports() df = fix_encodings(df) uri = f"{config['PATHS']['DATA']}airports.pickle" df.to_pickle(uri) log.info(f"Airport data exported to '{uri}'")
def train_sanity_check(tokenizer, predictions, target_id): # use the last sample in the batch predicted, target = detokenize( tokenizer, tf.squeeze(tf.argmax(predictions, axis=-1)[-1:]), tf.squeeze(target_id[:, :-1][-1:])) log.info(f'the true output_sequence is {target}') log.info(f'the predicted output_seq with teacher forcing is \ { predicted if predicted else "empty hence evaluation will be skipped"}' ) return predicted
def check_ckpt(checkpoint_path): ckpt = tf.train.Checkpoint(Model=Model, optimizer=optimizer) ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=10) if tf.train.latest_checkpoint(checkpoint_path): ckpt.restore(ckpt_manager.latest_checkpoint) log.info(ckpt_manager.latest_checkpoint + ' restored') else: log.info('No checkpoint found') return (ckpt_manager)
def run(self): # Store start time and task name self.name = self.__class__.__name__ self.t_data["name"] = self.name self.start_time = time.time() self.t_data["start_time"] = datetime.now().strftime( "%Y-%m-%d %H:%M:%S") # Run the task and store the resutls log.info(f"Starting {self.name}") self.run_std() self.save_result()
def check_ckpt(checkpoint_path): ckpt = tf.train.Checkpoint(Model=Model, optimizer=optimizer) ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=10) if tf.train.latest_checkpoint(checkpoint_path): ckpt.restore(ckpt_manager.latest_checkpoint) log.info(ckpt_manager.latest_checkpoint + ' restored') else: log.warning('No checkpoint found so using the initialized_weights') return ckpt_manager
def main( uri_in=f"{config['PATHS']['DATA']}flights/{date.today():%Y_%m_%d}/*.csv", uri_out=f"{config['PATHS']['DATA']}flights.parquet", ): """ Retreive airports """ spark = SparkSession.builder.getOrCreate() sdf = spark.read.option("header", "true").csv(uri_in) log.info(f"Merging {sdf.count()} rows of flights data") sdf.write.partitionBy("Inserted").parquet(uri_out, mode="overwrite") log.info(f"File '{uri_out}' exported")
def main(path=f"{config['PATHS']['DATA']}flights/{date.today():%Y_%m_%d}/"): """ Check data there is data in the flights folder """ spark = SparkSession.builder.getOrCreate() sdf = spark.read.option("header", "true").csv(f"{path}*.csv") n_rows = sdf.count() if n_rows == 0: msg = f"There is no data in {path}" log.info(msg) raise ValueError(msg) log.info(f"There are {n_rows} rows of flights data")
def save_url(url): blocked_urls = select(b.value for b in Block if b.type == "Url" and b.active)[:] if not any(x for x in blocked_urls if x in url): result = select(p for p in Url if p.url == url).count() if result == 0: if ".onion" in url: url_object = Url(url=url, date_added=datetime.now()) else: print("{} is blocked".format(url)) else: log.info("URL: {} is blocked".format(url)) commit()
def _embedding_from_bert(): with tf.device("CPU:0"): input_pretrained_bert = TFAutoModel.from_pretrained( config.input_pretrained_model, trainable=False, name=config.input_pretrained_model) target_pretrained_bert = TFAutoModel.from_pretrained( config.target_pretrained_model, trainable=False, name=config.target_pretrained_model ) if config['task'] == 'translate' else input_pretrained_bert decoder_embedding = target_pretrained_bert.get_weights()[0] log.info(f"Decoder_Embedding matrix shape '{decoder_embedding.shape}'") return (decoder_embedding, input_pretrained_bert, target_pretrained_bert)
def monitor_run(ckpt_save_path, bert_score, rouge_score, train_loss, step, to_monitor=config.monitor_metric): ckpt_fold, ckpt_string = os.path.split(ckpt_save_path) if config.run_tensorboard: with valid_output_sequence_writer.as_default(): tf.summary.scalar('ROUGE_f1', rouge_score, step=step) tf.summary.scalar('BERT_f1', bert_score, step=step) monitor_metrics = dict() monitor_metrics['BERT_f1'] = bert_score monitor_metrics['ROUGE_f1'] = rouge_score monitor_metrics['combined_metric'] = (monitor_metrics['BERT_f1'], monitor_metrics['ROUGE_f1']) # multiply with the weights monitor_metrics['combined_metric'] = round( tf.reduce_sum([(i * j) for i, j in zip( monitor_metrics['combined_metric'], config.combined_metric_weights) ]).numpy(), 2) log.info(f"combined_metric {monitor_metrics['combined_metric']:4f}") if config.last_recorded_value < monitor_metrics[to_monitor]: # reset tolerance to zero if the monitor_metric decreases before the tolerance threshold config.tolerance = 0 config.last_recorded_value = monitor_metrics[to_monitor] ckpt_files_tocopy = [files for files in os.listdir(os.path.split(ckpt_save_path)[0]) \ if ckpt_string in files] log.info( f'{to_monitor} is {monitor_metrics[to_monitor]:4f} so checkpoint files {ckpt_string} \ will be copied to best checkpoint directory') # copy the best checkpoints shutil.copy2(os.path.join(ckpt_fold, 'checkpoint'), config.best_ckpt_path) for files in ckpt_files_tocopy: shutil.copy2(os.path.join(ckpt_fold, files), config.best_ckpt_path) else: config.tolerance += 1 # stop if minimum training loss is reached if train_loss < config.min_train_loss: log.info(f'Stop training since minimum training loss reached') return False else: return True # Warn and early stop if config.tolerance > config.tolerance_threshold: log.warning('Tolerance exceeded') if config.early_stop: log.info( f'Early stopping since the {to_monitor} reached the tolerance threshold' ) return False else: return True else: return True
def change_dataset_and_train(addtional_tokens_per_batch, batch_size): memory_test_dataset = create_dataset( split='train', source_tokenizer=source_tokenizer, target_tokenizer=target_tokenizer, from_=90, to=100, shuffle=True, batch_size=batch_size ) log.info(f'Training with tokens_per_batch set to {addtional_tokens_per_batch}\ and batch_size set to {batch_size}') training_loop(memory_test_dataset.take(1000), False) gpu_usage = check_gpu_usage() log.info(f'GPU memory utilization is {gpu_usage}') return gpu_usage
def main(): """ Check that the airports passed are valid iata codes """ uri = f"{config['PATHS']['DATA']}airports.pickle" df = pd.read_pickle(uri) valid_codes = df["iata_code"].dropna().unique() airports = config["AIRPORTS"]["ORIGINS"].split(",") invalid_airports = [x for x in airports if x not in valid_codes] if invalid_airports: msg = f"Airports {invalid_airports} are not valid IATA codes" log.error(msg) raise ValueError(msg) log.info("All airports are valid IATA codes")
def start_bee(): """The start_bee function starts all the tasks related to starting the bee. When this function is called the bee will start working.""" log.debug("Bee has been started") while True: urls = get_urls() # update the url so different instances don't crawl the same url for url in urls: update_url(url) if len(urls) == 0: print("No URLs to be crawled, waiting for 60 seconds.") log.info('No URLs to be crawled, waiting for 60 seconds.') sleep(60) commit() continue connect_to_tor() for url in urls: try: content = get_content_from_url(url.url) print("{} is now being beeed".format(url.url)) content_hashed = hash_content(content) content_cleaned = clean_html(content) check_blocked = check_blocked_keywords(content_cleaned) if check_blocked is None: keywords = filter_keywords(content_cleaned) save_content(url.id, content_cleaned, content, content_hashed, keywords) else: print( "URL: " + url.url + " has been blocked because it contains a blocked keyword " + check_blocked) except (ValueError, NameError, TypeError) as error: log.error(str(error))
def eval_step(input_ids, target_ids_, target_ids, draft_mask, refine_mask): (draft_predictions, draft_attention_weights, refine_predictions, refine_attention_weights) = Model(input_ids, target_ids_, False) draft_output_sequence_loss = loss_function(target_ids[:, 1:, :], draft_predictions, draft_mask) if config.use_refine_decoder: refine_output_sequence_loss = loss_function(target_ids[:, :-1, :], refine_predictions, refine_mask) else: refine_output_sequence_loss = 0 regularization_loss = tf.add_n(Model.losses) loss = draft_output_sequence_loss + refine_output_sequence_loss + regularization_loss log.info(Model.summary()) if config.save_initial_weights: initial_weights = os.path.join(config.initial_weights, 'initial_weights') Model.save_weights(initial_weights) return loss
def save_evaluate_monitor(ck_pt_mgr, val_dataset, target_tokenizer, predictions, target_ids, step, start_time, return_attention=False): ckpt_save_path = ck_pt_mgr.save() # print the detokenized training output of a single sample predicted = train_sanity_check(target_tokenizer, predictions, target_ids, log) evaluate = train_loss.result( ) < config.start_evaluate_when and True if predicted else False # Run evaluation only if the predictions made by the teacher forced output is not empty # and the train_loss is lesser than start_evaluate_when if evaluate: (task_score, bert_score, draft_attention_weights, refine_attention_weights) = evaluate_validation_set( val_dataset, step, return_with_attention_weights=return_attention) early_stop_training = monitor_eval_metrics(ckpt_save_path, bert_score, task_score, train_loss.result(), step, log, config) else: log.info( 'Not running evaluation since loss is not lesser than config.start_evaluate_when' ) (task_score, bert_score) = (0, 0) early_stop_training = False draft_attention_weights = None refine_attention_weights = None training_results(step, train_loss.result(), train_accuracy.result(), task_score, bert_score, (time.time() - start_time), ckpt_save_path, log, config) train_loss.reset_states() train_accuracy.reset_states() return (early_stop_training, draft_attention_weights, refine_attention_weights)
async def main(loop): log.debug('scout has been started') while True: urls = get_urls_from_database() # update urls immediately to avoid different instances crawling the same urls for url in urls: update_url(url) if len(urls) == 0: print("No URLs to be crawled, waiting for 60 seconds.") log.info('No URLs to be crawled, waiting for 60 seconds.') sleep(60) commit() continue results = await tor.get_content_from_urls(loop, urls) urls_from_content = get_urls_from_results(urls, results) for u in urls_from_content: if u is not None: save_url(u) print('Found ', len(urls_from_content), ' urls')
def training_loop(dataset, check_model_capacity, detokenize_samples=None): min_loss = 10000000 if check_model_capacity: dataset = dataset.repeat(670) for (step, (input_ids, target_ids)) in tqdm(enumerate(dataset, 1), initial=1): start=time.time() grad_accum_flag = (True if ((step)%config.gradient_accumulation_steps) == 0 else False) if config.accumulate_gradients else None predictions = train_step( input_ids, target_ids, grad_accum_flag ) if grad_accum_flag is not None: if grad_accum_flag: if (step)%config.steps_to_print_training_info==0: predicted_ids = train_sanity_check(target_tokenizer, predictions, target_ids) train_loss = batch_run_check( step, start ) else: if (step)%config.steps_to_print_training_info==0: train_loss = batch_run_check( step, start ) if check_model_capacity: if min_loss > train_loss: min_loss = train_loss else: log.warning('Loss not decreasing watch out') monitor_early_stop = monitor_run( 'not saving', 0, 0, 0.0, 1, copy_best_ckpt=False ) if check_model_capacity: log.info(f'target_ids are {target_ids}') log.info(f'predicted ids are {predicted_ids}') if train_loss < config.min_train_loss: log.info('Minimum training loss reached') else: log.info("Loss didn't reach upto the min_train_loss specified, try to increase\ the parameters of the model or number of train steps")
def create_dataset(split, source_tokenizer, target_tokenizer, from_, to, batch_size, buffer_size=None, use_tfds=True, csv_path=None, drop_remainder=False, num_examples_to_select=config.samples_to_test): examples, metadata = tfds.load('ted_hrlr_translate/pt_to_en', with_info=True, as_supervised=True) train_examples, val_examples = examples['train'], examples['validation'] #initialize the first dataset to the train_examples variable #Concatenate all the train datasets if split == 'train': raw_dataset = train_examples elif split == 'validation': raw_dataset = val_examples tf_dataset = raw_dataset.map(tf_encode(source_tokenizer, target_tokenizer, config.input_seq_length, config.target_seq_length), num_parallel_calls=AUTOTUNE) tf_dataset = tf_dataset.filter(filter_max_length) tf_dataset = tf_dataset.take(num_examples_to_select) tf_dataset = tf_dataset.cache() if buffer_size: tf_dataset = tf_dataset.shuffle(buffer_size, seed=100) tf_dataset = tf_dataset.padded_batch(batch_size, padded_shapes=([-1], [-1]), drop_remainder=drop_remainder) tf_dataset = tf_dataset.prefetch(buffer_size=AUTOTUNE) log.info(f'{split} tf_dataset created') return tf_dataset
def create_vocab(tokenizer_path, tok_type): try: tokenizer = tfds.features.text.SubwordTextEncoder.load_from_file(config.tokenizer_path) except FileNotFoundError: log.warning(f'Vocab files not available in {config.tokenizer_path} so building it from the training set') if config.use_tfds: examples, metadata = tfds.load(config.tfds_name, with_info=True, as_supervised=True) train_examples = examples['train'] if tok_type=='source': tokenizer = tfds.features.text.SubwordTextEncoder.build_from_corpus( (ip_seq.numpy() for ip_seq, _ in train_examples), target_vocab_size=2**13) else: tokenizer = tfds.features.text.SubwordTextEncoder.build_from_corpus( (op_seq.numpy() for _, op_seq in train_examples), target_vocab_size=2**13) tokenizer.save_to_file(config.tokenizer_path) if tok_type=='source': assert(tokenizer.vocab_size+2 == config.input_vocab_size),f'{tok_type}vocab size in configuration script should be {tokenizer.vocab_size+2}' else: assert(tokenizer.vocab_size+2 == config.output_vocab_size),f'{tok_type}vocab size in configuration script should be {tokenizer.vocab_size+2}' log.info(f'{tok_type} vocab file created and saved to {config.tokenizer_path}') return tokenizer
def training_results(step, rouge_score, bert_score, timing_info, ckpt_save_path): log.info( model_metrics.format(step, train_loss.result(), train_accuracy.result(), rouge_score, bert_score)) log.info(evaluation_step.format(step, timing_info)) log.info(checkpoint_details.format(step, ckpt_save_path))
def create_dataset(split, source_tokenizer, target_tokenizer, from_, to, batch_size, shuffle=None, use_tfds=True, drop_remainder=False, num_examples_to_select=config.samples_to_test): raw_dataset, ds_info = tfds.load(config.tfds_name, with_info=True, as_supervised=True, data_dir=config.tfds_data_dir, builder_kwargs=config.tfds_data_version, split=tfds.core.ReadInstruction( split, from_=from_, to=to, unit='%')) tf_dataset = raw_dataset.map(tf_encode(source_tokenizer, target_tokenizer, config.input_seq_length, config.target_seq_length), num_parallel_calls=AUTOTUNE) tf_dataset = tf_dataset.filter(filter_max_length) tf_dataset = tf_dataset.take(num_examples_to_select) tf_dataset = tf_dataset.cache() if shuffle: buffer_size = (sum( [i.num_examples for i in list(ds_info.splits.values())])) tf_dataset = tf_dataset.shuffle(buffer_size, seed=100) tf_dataset = tf_dataset.padded_batch(batch_size, padded_shapes=([-1], [-1]), drop_remainder=drop_remainder) tf_dataset = tf_dataset.prefetch(buffer_size=AUTOTUNE) log.info(f'{split} tf_dataset created') return tf_dataset
def main(): """ Get all flights of each pair and export them as a csv for each pair """ path_raw = config["PATHS"]["DATA"] + f"flights/{date.today():%Y_%m_%d}/" # Create folder os.makedirs(path_raw, exist_ok=True) airports_pairs = get_pairs() total_pairs = len(airports_pairs) for i, (origin, dest) in enumerate(airports_pairs): log.info( f"Quering flights from '{origin}' to '{dest}' ({i + 1}/{total_pairs})" ) df = query_pair(origin, dest) if df is not None: uri = f"{path_raw}{origin}_{dest}.csv" df.to_csv(uri, index=False) log.debug(f"Exporting '{uri}'")
def batch_run_check(batch, start): if config.run_tensorboard: with train_output_sequence_writer.as_default(): tf.summary.scalar('train_loss', train_loss.result(), step=batch) tf.summary.scalar('train_accuracy', train_accuracy.result(), step=batch) if config.display_model_summary: log.info(Model.summary()) log.info(batch_zero.format(time.time() - start)) config.display_model_summary = False log.info( batch_run_details.format(train_loss.result(), train_accuracy.result())) return train_loss.result()
def batch_run_check(batch, start_time, bert_f1_score): if config.run_tensorboard: with train_output_sequence_writer.as_default(): tf.summary.scalar('train_loss', train_loss.result(), step=batch) if config.display_model_summary: log.info(Model.summary()) log.info(batch_zero.format(time.time() - start_time)) config.display_model_summary = False log.info( batch_run_details.format( tf.debugging.assert_all_finite(train_loss.result(), message="NaN's or Inf's.", name='NAN_assertion'), bert_f1_score.numpy()))
top_k=config.top_k): batch_size = tf.shape(input_ids)[0] if training is not None: return self.fit(input_ids, target_ids, training, look_ahead_mask, dec_padding_mask, batch_size) else: return self.predict(input_ids, batch_size=batch_size, draft_decoder_type=decoder_type, beam_size=beam_size, length_penalty=length_penalty, temperature=temperature, top_p=top_p, top_k=top_k) Model = Bertified_transformer( num_layers=config.num_layers, d_model=config.d_model, num_heads=config.num_heads, dff=config.dff, input_vocab_size=config.input_vocab_size, target_vocab_size=config.target_vocab_size, add_pointer_generator=config.add_pointer_generator) if config.print_config: if config['add_bias']: config['add_bias'] = True log.info(f'Configuration used \n {config}')
def create_dataset(split, source_tokenizer, target_tokenizer, from_, to, batch_size, shuffle=None, drop_remainder=False, num_examples_to_select=config.samples_to_train): if config.tfds_name == 'en_tam_parallel_text': en_tam_ds = defaultdict(list) record_count = 0 #List of available datasets in the package dataset_names = [ 'GNOME_v1_en_to_ta', 'GNOME_v1_en_AU_to_ta', 'GNOME_v1_en_CA_to_ta', 'GNOME_v1_en_GB_to_ta', 'GNOME_v1_en_US_to_ta', 'KDE4_v2_en_to_ta', 'KDE4_v2_en_GB_to_ta', 'Tatoeba_v20190709_en_to_ta', 'Ubuntu_v14.10_en_to_ta_LK', 'Ubuntu_v14.10_en_GB_to_ta_LK', 'Ubuntu_v14.10_en_AU_to_ta_LK', 'Ubuntu_v14.10_en_CA_to_ta_LK', 'Ubuntu_v14.10_en_US_to_ta_LK', 'Ubuntu_v14.10_en_to_ta', 'Ubuntu_v14.10_en_GB_to_ta', 'Ubuntu_v14.10_en_AU_to_ta', 'Ubuntu_v14.10_en_CA_to_ta', 'Ubuntu_v14.10_en_NZ_to_ta', 'Ubuntu_v14.10_en_US_to_ta', 'OpenSubtitles_v2018_en_to_ta', 'OpenSubtitles_v2016_en_to_ta', 'en_ta', 'github_joshua_en_ta' ] for name in dataset_names: en_tam_ds[(name, 'metadata_' + name)] = tfds.load( f'{config.tfds_name}/' + name, with_info=True, as_supervised=True, data_dir=config.tfds_data_dir, builder_kwargs={'version': config.tfds_data_version}, ) for i, j in en_tam_ds.keys(): record_count += (sum([ i.num_examples for i in list(en_tam_ds[(i, j)][1].splits.values()) ])) if not config.test_script: log.info(f'Total record count without filtering is {record_count}') #initialize the first dataset to the train_examples variable #Concatenate all the train datasets if split == 'train': raw_dataset = en_tam_ds[('GNOME_v1_en_to_ta', 'metadata_GNOME_v1_en_to_ta')][0]['train'] for typ in list(en_tam_ds.keys())[1:]: raw_dataset = raw_dataset.concatenate( en_tam_ds[typ][0]['train']) #other splits include validation and test else: raw_dataset = en_tam_ds[('en_ta', 'metadata_en_ta')][0][split] else: raw_dataset, ds_info = tfds.load( config.tfds_name, with_info=True, as_supervised=True, data_dir=config.tfds_data_dir, builder_kwargs={'version': config.tfds_data_version}, split=tfds.core.ReadInstruction(split, from_=from_, to=to, unit='%')) record_count = (sum( [i.num_examples for i in list(ds_info.splits.values())])) tf_dataset = raw_dataset.map(tf_encode(source_tokenizer, target_tokenizer), num_parallel_calls=parallel_calls) tf_dataset = tf_dataset.filter(filter_max_length) if config.model == 'bertified_transformer': tf_dataset = tf_dataset.map(tf_pad_encoded_ids) tf_dataset = tf_dataset.take(num_examples_to_select) tf_dataset = tf_dataset.cache() if shuffle: tf_dataset = tf_dataset.shuffle(record_count, seed=100) tf_dataset = tf_dataset.padded_batch(batch_size, padded_shapes=([-1], [-1]), drop_remainder=drop_remainder) tf_dataset = tf_dataset.prefetch(buffer_size=parallel_calls) log.info(f'{split} tf_dataset created') return tf_dataset
(task_score, bert_score, draft_attention_weights, refine_attention_weights) = evaluate_validation_set(test_dataset, step, decoder_type, beam_size, length_penalty, temperature, top_p, top_k) combined_metric = (0.8*bert_score) + (0.2*task_score) if max_combined_metric < combined_metric: max_combined_metric = combined_metric best_beam_size = beam_size best_length_penalty = length_penalty best_temperature = temperature best_top_p = top_p best_top_k = top_k log.info(infer_template.format(task_score, bert_score, combined_metric, beam_size, step)) log.info(best_combo.format( best_beam_size, best_length_penalty, best_temperature, best_top_p, best_top_k, max_combined_metric ) )