def __init__(self, embed_matrix, dictionary: dict, **kwargs): """ Parameters ---------- embed_matrix: numpy array dictionary: dictionary """ self._embed_matrix = embed_matrix self._dictionary = dictionary self._reverse_dictionary = {v: k for k, v in dictionary.items()} self.words = list(dictionary.keys()) self._jarowinkler = JaroWinkler() device = get_device(**kwargs) _graph = tf.Graph() with _graph.as_default(): with tf.device(device): self._embedding = tf.compat.v1.placeholder( tf.float32, self._embed_matrix.shape) self._x = tf.compat.v1.placeholder( tf.float32, [None, self._embed_matrix.shape[1]]) normed_embedding = tf.nn.l2_normalize(self._embedding, axis=1) normed_array = tf.nn.l2_normalize(self._x, axis=1) self._cosine_similarity = tf.matmul( normed_array, tf.transpose(normed_embedding, [1, 0])) self._sess = generate_session(_graph, **kwargs)
def __init__(self, xlnet_config, tokenizer, checkpoint, pool_mode='last', **kwargs): kwargs_config = dict( is_training=True, use_tpu=False, use_bfloat16=False, dropout=0.0, dropatt=0.0, init='normal', init_range=0.1, init_std=0.05, clamp_len=-1, ) xlnet_parameters = xlnet_lib.RunConfig(**kwargs_config) self._tokenizer = tokenizer device = get_device(**kwargs) _graph = tf.Graph() with _graph.as_default(): with tf.device(device): self.X = tf.placeholder(tf.int32, [None, None]) self.segment_ids = tf.placeholder(tf.int32, [None, None]) self.input_masks = tf.placeholder(tf.float32, [None, None]) xlnet_model = xlnet_lib.XLNetModel( xlnet_config=xlnet_config, run_config=xlnet_parameters, input_ids=tf.transpose(self.X, [1, 0]), seg_ids=tf.transpose(self.segment_ids, [1, 0]), input_mask=tf.transpose(self.input_masks, [1, 0]), ) self.logits = xlnet_model.get_pooled_out(pool_mode, True) self._sess = generate_session(_graph, **kwargs) self._sess.run(tf.global_variables_initializer()) tvars = tf.trainable_variables() assignment_map, _ = get_assignment_map_from_checkpoint( tvars, checkpoint) self._saver = tf.train.Saver(var_list=assignment_map) attentions = [ n.name for n in tf.get_default_graph().as_graph_def().node if 'rel_attn/Softmax' in n.name ] g = tf.get_default_graph() self.attention_nodes = [ g.get_tensor_by_name('%s:0' % (a)) for a in attentions ]
def pagerank(array, fast_pagerank=True, retry=5, **kwargs): device = get_device(**kwargs) cpu = False fail = True if 'GPU' in device: try: import cugraph cpu = True except Exception as e: msg = ( 'cugraph not installed. Please install it from https://github.com/rapidsai/cugraph. \n' 'Will calculate pagerank using networkx CPU version.') logging.warning(msg) cpu = True else: cpu = True if cpu: if fast_pagerank: from scipy import sparse from malaya.graph import fast_pagerank G = sparse.csr_matrix(array) r = fast_pagerank.pagerank(G) scores = {i: r[i] for i in range(len(r))} fail = False else: nx_graph = nx.from_numpy_array(array) for _ in range(retry): try: scores = nx.pagerank(nx_graph, max_iter=10000) fail = False break except Exception as e: logging.warning(e) if fail: raise Exception( 'pagerank not able to converge, rerun may able to solve it.') return scores
def __init__(self, hparams, encoder, generate_length, temperature, top_k, **kwargs): self._encoder = encoder device = get_device(**kwargs) self._graph = tf.Graph() with self._graph.as_default(): with tf.device(device): self._X = tf.placeholder(tf.int32, [1, None]) self._model = sample_sequence( hparams=hparams, length=generate_length, context=self._X, batch_size=1, temperature=temperature, top_k=top_k, ) self._sess = generate_session(self._graph, **kwargs) self._sess.run(tf.global_variables_initializer()) self._saver = tf.train.Saver(tf.trainable_variables())
def __init__(self, bert_config, tokenizer, **kwargs): device = get_device(**kwargs) _graph = tf.Graph() with _graph.as_default(): with tf.device(device): self.X = tf.placeholder(tf.int32, [None, None]) self.segment_ids = tf.placeholder(tf.int32, [None, None]) self.top_p = tf.placeholder(tf.float32, None) self.top_k = tf.placeholder(tf.int32, None) self.k = tf.placeholder(tf.int32, None) self.temperature = tf.placeholder(tf.float32, None) self.indices = tf.placeholder(tf.int32, [None, None]) self.MASK = tf.placeholder(tf.int32, [None, None]) self._tokenizer = tokenizer self.model = modeling.AlbertModel( config=bert_config, is_training=False, input_ids=self.X, input_mask=self.MASK, use_one_hot_embeddings=False, ) self.logits = self.model.get_pooled_output() input_tensor = self.model.get_sequence_output() output_weights = self.model.get_embedding_table() with tf.variable_scope('cls/predictions'): with tf.variable_scope('transform'): input_tensor = tf.layers.dense( input_tensor, units=bert_config.embedding_size, activation=modeling.get_activation( bert_config.hidden_act), kernel_initializer=modeling.create_initializer( bert_config.initializer_range), ) input_tensor = modeling.layer_norm(input_tensor) output_bias = tf.get_variable( 'output_bias', shape=[bert_config.vocab_size], initializer=tf.zeros_initializer(), ) logits = tf.matmul(input_tensor, output_weights, transpose_b=True) self._logits = tf.nn.bias_add(logits, output_bias) self._log_softmax = tf.nn.log_softmax(self._logits, axis=-1) logits = tf.gather_nd(self._logits, self.indices) logits = logits / self.temperature def necleus(): return top_p_logits(logits, self.top_p) def select_k(): return top_k_logits(logits, self.top_k) logits = tf.cond(self.top_p > 0, necleus, select_k) self.samples = tf.multinomial(logits, num_samples=self.k, output_dtype=tf.int32) self._sess = generate_session(_graph, **kwargs) self._sess.run(tf.global_variables_initializer()) var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='bert') var_lists.extend( tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='cls')) self._saver = tf.train.Saver(var_list=var_lists) attns = _extract_attention_weights( bert_config.num_hidden_layers, tf.get_default_graph()) self.attns = attns
def __init__(self, num_unique_documents, vocab_size, num_topics, freqs, embedding_size=128, num_sampled=40, learning_rate=1e-3, lmbda=150.0, alpha=None, power=0.75, batch_size=32, clip_gradients=5.0, **kwargs): device = get_device(**kwargs) _graph = tf.Graph() with _graph.as_default(): with tf.device(device): moving_avgs = tf.train.ExponentialMovingAverage(0.9) self.batch_size = batch_size self.freqs = freqs self.X = tf.placeholder(tf.int32, shape=[None]) self.Y = tf.placeholder(tf.int64, shape=[None]) self.DOC = tf.placeholder(tf.int32, shape=[None]) self.switch_loss = tf.Variable(0, trainable=False) train_labels = tf.reshape(self.Y, [-1, 1]) sampler = tf.nn.fixed_unigram_candidate_sampler( train_labels, num_true=1, num_sampled=num_sampled, unique=True, range_max=vocab_size, distortion=power, unigrams=self.freqs, ) self.word_embedding = tf.Variable( tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0)) self.nce_weights = tf.Variable( tf.truncated_normal( [vocab_size, embedding_size], stddev=tf.sqrt(1 / embedding_size), )) self.nce_biases = tf.Variable(tf.zeros([vocab_size])) scalar = 1 / np.sqrt(num_unique_documents + num_topics) self.doc_embedding = tf.Variable( tf.random_normal( [num_unique_documents, num_topics], mean=0, stddev=50 * scalar, )) self.topic_embedding = tf.get_variable( 'topic_embedding', shape=[num_topics, embedding_size], dtype=tf.float32, initializer=tf.orthogonal_initializer(gain=scalar), ) pivot = tf.nn.embedding_lookup(self.word_embedding, self.X) proportions = tf.nn.embedding_lookup(self.doc_embedding, self.DOC) doc = tf.matmul(proportions, self.topic_embedding) doc_context = doc word_context = pivot context = tf.add(word_context, doc_context) loss_word2vec = tf.reduce_mean( tf.nn.nce_loss( weights=self.nce_weights, biases=self.nce_biases, labels=self.Y, inputs=context, num_sampled=num_sampled, num_classes=vocab_size, num_true=1, sampled_values=sampler, )) self.fraction = tf.Variable(1, trainable=False, dtype=tf.float32) n_topics = self.doc_embedding.get_shape()[1].value log_proportions = tf.nn.log_softmax(self.doc_embedding) if alpha is None: alpha = 1.0 / n_topics loss = (alpha - 1) * log_proportions prior = tf.reduce_sum(loss) loss_lda = lmbda * self.fraction * prior global_step = tf.Variable(0, trainable=False, name='global_step') self.cost = tf.cond( global_step < self.switch_loss, lambda: loss_word2vec, lambda: loss_word2vec + loss_lda, ) loss_avgs_op = moving_avgs.apply( [loss_lda, loss_word2vec, self.cost]) with tf.control_dependencies([loss_avgs_op]): optimizer = tf.train.AdamOptimizer( learning_rate=learning_rate) gvs = optimizer.compute_gradients(self.cost) capped_gvs = [( tf.clip_by_value(grad, -clip_gradients, clip_gradients), var, ) for grad, var in gvs] self.optimizer = optimizer.apply_gradients(capped_gvs) self.sess = generate_session(_graph, **kwargs) self.sess.run(tf.global_variables_initializer())