def combined_loss(y_pred, y):
"""
This receives a combination of the predictions and combination of ground truths
The ground truth is not sequential, but a single number. This is therefore repeated, to provide a label
for every intermediate step.
Returns the means of the losses.
:param y_pred:
:param y:
:return: (combined_loss, relevance_loss, diversity_loss, context_loss)
"""
item_logits = y_pred.logits
context_logits = y_pred.context_logits
item_y, context_y = split_input(y,
requires_grad=[False, False],
dtype=[torch.long, torch.float],
device=device)
relevance_loss = cross_entropy(item_logits, item_y, reduction='none')
context_loss = loss_properties(context_logits, context_y)
# Linearly combine
combined_loss = lambda_multitask_loss * relevance_loss + (
1 - lambda_multitask_loss) * context_loss
# Calculate means
combined_loss = combined_loss.mean(-1)
relevance_loss = relevance_loss.mean(-1)
context_loss = context_loss.mean(-1)
return (combined_loss, relevance_loss, context_loss)
def forward(self, inputs):
item_x, context_x, user_id = split_input(inputs, requires_grad=False,
dtype=[torch.long, torch.float, torch.long], device=self.device)
batch_size = item_x.shape[0]
(item_enc_hidden, ctxt_enc_last_hidden, norm_attn) = self.encode(item_x, context_x, user_id)
global_preference = self.concat_to_item_tfm(item_enc_hidden)
# calculate p(v | sequence) based on the global preference
# (batch x num_item_units) times (|V| x num_item_units) -> (batch x |V|)
item_logits = torch.einsum('bd,vd->bv', global_preference, self.item_embedding.weight)
if not self.training:
item_probs = item_logits.softmax(dim=-1)
if self.lambda_score == 1:
_, recommended_list = torch.topk(item_probs, self.recommendation_len, sorted=True)
else:
# DIVERSITY WITH Maximal Marginal Relevance
init_preds = item_probs.argmax(dim=-1)
recommended_list = init_preds.detach().view(batch_size, 1)
for _ in range(self.recommendation_len - 1):
# calculate diversity using the embeddings of the given ids
selected = self.compute_distance(item_probs, recommended_list)
recommended_list = torch.cat([recommended_list, selected.view(batch_size, 1)], dim=-1)
else:
recommended_list = torch.zeros(1, device=self.device)
# return as a tuple for the trainer. This will be split up in the loss function.
recommended_list = recommended_list.requires_grad_(False)
return Output(preds=recommended_list, logits=item_logits,
attention=norm_attn, context_logits=None, context_preds=None)
def update(self, output):
y_pred, y = output
recommended_list = y_pred.preds
item_logits = y_pred.logits
context_logits = y_pred.context_logits
context_predictions = y_pred.context_preds
attention = y_pred.attention
if context_logits is not None:
item_y, context_y = split_input(y,
dtype=torch.int,
requires_grad=False,
device=device)
else:
item_y, _ = split_input(y,
dtype=torch.int,
requires_grad=False,
device=device)
item_probs = item_logits.softmax(dim=-1)
item_entropy = Categorical(probs=item_probs).entropy()
self._statistics[ITEM_ENTROPY].extend(item_entropy)
self._statistics[ITEM_MIN].extend(item_probs.min(dim=-1).values)
self._statistics[ITEM_MAX].extend(item_probs.max(dim=-1).values)
self._prediction_lists.extend(
recommended_list
) # extend because we then add the recommendation list
self._labels.extend(item_y)
if attention is not None:
self._attentions.extend(attention)
if context_predictions is not None:
context_probs = context_logits.softmax(dim=-1)
context_entropy = Categorical(probs=context_probs).entropy()
self._statistics[CONTEXT_ENTROPY].extend(context_entropy)
self._statistics[CONTEXT_MIN].extend(
context_probs.min(dim=-1).values)
self._statistics[CONTEXT_MAX].extend(
context_probs.max(dim=-1).values)
self._context_predictions.extend(context_predictions)
self._context_labels.extend(context_y)
self._context_probs.extend(context_probs)
def DSR_loss(y_pred, y):
item_logits = y_pred.logits
item_y, _ = split_input(y,
requires_grad=[False, False],
dtype=torch.long,
device=device)
# Calculate relevance loss
relevance_loss = cross_entropy(item_logits, item_y, reduction='mean')
return (relevance_loss, relevance_loss)
def forward(self, inputs):
item_x, context_x, user_id = split_input(
inputs,
requires_grad=False,
dtype=[torch.long, torch.float, torch.long],
device=self.device)
batch_size = item_x.shape[0]
(item_enc_hidden, ctxt_enc_hidden,
norm_attn) = self.encode(item_x, context_x, user_id)
context_predict_input = torch.cat((item_enc_hidden, ctxt_enc_hidden),
dim=-1)
context_logits = self.context_MLP(context_predict_input)
context_probs = context_logits.softmax(dim=-1)
context_preds = context_logits.argmax(dim=-1)
context_y_embedded = torch.einsum('bc, cd -> bd', context_probs,
self.context_embedding)
context_y_embedded = dropout(context_y_embedded,
p=self.dropout_rate,
training=self.training)
global_preference = self.bilinear_mapping(item_enc_hidden,
context_y_embedded)
# calculate p(v | sequence) based on the global preference
item_logits = torch.einsum('bd,vd->bv', global_preference,
self.item_embedding.weight)
if not self.training:
item_probs = item_logits.softmax(dim=-1)
recommended_list = []
init_preds = item_probs.argmax(dim=-1)
recommended_list.append(init_preds.detach())
satisfactions = context_probs
for _ in range(self.recommendation_len - 1):
# calculate diversity using the embeddings of the given ids
# selected = self.compute_distance(item_probs, recommended_list)
selected, satisfactions = self.compute_highest_marginal_utility(
item_probs, satisfactions, recommended_list)
recommended_list.append(selected)
recommended_list = torch.stack(recommended_list, 1)
else:
recommended_list = torch.zeros(1, device=self.device)
# return as a tuple for the trainer. This will be split up in the loss function.
recommended_list = recommended_list.requires_grad_(False)
context_preds = context_preds.requires_grad_(False)
return Output(preds=recommended_list,
logits=item_logits,
attention=norm_attn,
context_logits=context_logits.view(batch_size, -1),
context_preds=context_preds)
def forward(self, inputs):
item_x, context_x, user_id = split_input(
inputs,
requires_grad=False,
dtype=[torch.long, torch.float, torch.long],
device=self.device)
batch_size = item_x.shape[0]
(item_enc_hidden, ctxt_enc_hidden,
norm_attn) = self.encode(item_x, context_x, user_id)
# user_embeddings = self.user_embedding(user_id) # (batch x num_user_units)
context_predict_input = torch.cat((item_enc_hidden, ctxt_enc_hidden),
dim=-1)
context_logits = self.context_MLP(context_predict_input)
context_probs = context_logits.softmax(dim=-1)
context_preds = context_logits.argmax(dim=-1)
context_y_embedded = torch.einsum('bc, cd -> bd', context_probs,
self.context_embedding)
context_y_embedded = dropout(context_y_embedded,
p=self.dropout_rate,
training=self.training)
global_preference = self.bilinear_mapping(item_enc_hidden,
context_y_embedded)
# calculate p(v | sequence) based on the global preference
item_logits = torch.einsum('bd,vd->bv', global_preference,
self.item_embedding.weight)
if not self.training:
item_probs = item_logits.softmax(dim=-1)
_, recommended_list = torch.topk(
item_probs,
self.recommendation_len,
sorted=True,
)
else:
recommended_list = torch.zeros(1, device=self.device)
# return as a tuple for the trainer. This will be split up in the loss function.
recommended_list = recommended_list.requires_grad_(False)
context_preds = context_preds.requires_grad_(False)
return Output(preds=recommended_list,
logits=item_logits,
attention=norm_attn,
context_logits=context_logits,
context_preds=context_preds)
def run_command(command):
c = split_input(command)
name = c['cmd']
if name == "exit":
exit()
elif name == "help":
help(c['fun'])
elif name == "run":
run(c)
elif name == "stop":
stop()
elif name == "test":
test()
elif name == "runsweep":
run_sweep()
else:
println("Unknown command: %s, try the `help` command" % name)
def train(self, xs, ys):
"""
train model
:param xs: dataset xs
:param ys: dataset ys
:return: loss
train op
global step
tensorflow summary
"""
tower_grads = []
global_step = tf.train.get_or_create_global_step()
global_step_ = global_step * self.hp.gpu_nums
lr = noam_scheme(self.hp.d_model, global_step_, self.hp.warmup_steps)
optimizer = tf.train.AdamOptimizer(lr)
losses = []
xs, ys = split_input(xs, ys, self.hp.gpu_nums)
with tf.variable_scope(tf.get_variable_scope()):
for no in range(self.hp.gpu_nums):
with tf.device("/gpu:%d" % no):
with tf.name_scope("tower_%d" % no):
memory_h, memory_u, sents1 = self.encode(xs[no])
logits, y, sents2 = self.decode(
xs[no], ys[no], memory_h, memory_u)
tf.get_variable_scope().reuse_variables()
loss = self._calc_loss(y, logits)
losses.append(loss)
grads = optimizer.compute_gradients(loss)
# print(grads)
tower_grads.append(grads)
with tf.device("/cpu:0"):
grads = self.average_gradients(tower_grads)
train_op = optimizer.apply_gradients(grads,
global_step=global_step)
loss = sum(losses) / len(losses)
tf.summary.scalar('lr', lr)
tf.summary.scalar("train_loss", loss)
summaries = tf.summary.merge_all()
return loss, train_op, global_step_, summaries
def main():
args = sys.argv
utils.validate_input(args)
sub_matrix_path, seqs_paths = utils.split_input(args)
sub_matrix = utils.read_scoring_matrix(sub_matrix_path)
alphabet = utils.read_alphabet(sub_matrix_path)
seqs_dict = utils.get_seqs(seqs_paths)
i = 0
for id1, seq1 in seqs_dict.items():
j = 0
for id2, seq2 in seqs_dict.items():
if i < j:
timestart = datetime.now()
hsps = utils.find_hsps(k, T, sub_matrix, seq1, seq2, alphabet)
msps = utils.extend_hsps_to_msps(hsps, sub_matrix, seq1, seq2, X)
graph = utils.gen_graph(msps)
score = utils.runDAG(graph)
timedelta = datetime.now() - timestart
utils.update_output_file(id1, id2, score)
print(f"({id1}, {id2}):\t{timedelta}\t msps: {len(msps)}")
j += 1
i += 1
from env import api_id, api_hash, bot_token, TG_DOWNLOAD_PATH, TG_DOWNLOAD_PATH_TORRENTS, TG_UNZIP_TORRENTS, \ YOUTUBE_LINKS_SOPORTED, TG_DOWNLOAD_PATH, SESSION, \ TG_MAX_PARALLEL, TG_DL_TIMEOUT, PATH_TMP, PATH_COMPLETED, TG_PROGRESS_DOWNLOAD, \ YOUTUBE_FORMAT, TG_DOWNLOAD_PATH, PATH_YOUTUBE from logger import logger from utils import splash, create_directory, getDownloadPath, getUsers, split_input, config_file from youtube import youtube_download session = SESSION download_path = TG_DOWNLOAD_PATH download_path_torrent = TG_DOWNLOAD_PATH_TORRENTS # Directorio bajo vigilancia de DSDownload u otro. AUTHORIZED_USER, usuarios = getUsers() youtube_list = split_input(YOUTUBE_LINKS_SOPORTED) queue = asyncio.Queue() number_of_parallel_downloads = TG_MAX_PARALLEL maximum_seconds_per_download = TG_DL_TIMEOUT # Directorio temporal tmp_path = PATH_TMP completed_path = PATH_COMPLETED temp_completed_path = '' create_directory(tmp_path) create_directory(completed_path) FOLDER_GROUP = ''
