def prepare_train(): global tokenizer print("prepare training data") with FileIO(os.path.join(FLAGS.buckets, "texts.pkl"), mode='r+') as f: texts = pickle.load(f)[:25000] with FileIO(os.path.join(FLAGS.buckets, "texts_unsup.pkl"), mode='r+') as f: texts += pickle.load(f) tokenizer = Tokenizer(num_words=vocab_size) tokenizer.fit_on_texts(texts) sequence = tokenizer.texts_to_sequences(texts) sum_words = sum([len(seq) for seq in sequence]) print('there are %d words' % (sum_words)) x = np.zeros((sum_words, 1), dtype=np.int32) y = np.zeros((sum_words, 1), dtype=np.int32) index = 0 for i, seq in enumerate(sequence): for s in seq: x[index] = i y[index] = s index += 1 indice = np.arange(sum_words) np.random.shuffle(indice) x = x[indice] y = y[indice] return x, y, sum_words
def get_input(): with FileIO(os.path.join(FLAGS.buckets, "20news/texts.pkl"), mode='r+') as f: texts = pickle.load(f) tokenizer = Tokenizer(num_words=num_words) tokenizer.fit_on_texts(texts[:num_train]) sequences = tokenizer.texts_to_sequences(texts) sequences_reverse=[list(reversed(seq)) for seq in sequences] x = pad_sequences(sequences, maxlen=max_len) x_reverse=pad_sequences(sequences_reverse, maxlen=max_len) word_index = tokenizer.word_index embeddings_index = {} wordX = np.load(FileIO(os.path.join(FLAGS.buckets, "glove/embedding.300d.npy"),mode='r+')) allwords = pickle.load(FileIO(os.path.join(FLAGS.buckets, "glove/words.pkl"),mode='r+')) print(len(allwords)) for i in range(len(allwords)): embeddings_index[allwords[i]] = wordX[i, :] embedding_matrix = np.zeros((num_words, 300)) for word, i in word_index.items(): embedding_vector = embeddings_index.get(word) if embedding_vector is not None and i < num_words: embedding_matrix[i] = embedding_vector x_train_0 = x[:num_train] x_train_1 = x_reverse[:num_train] x_test_0 = x[num_train:] x_test_1 = x_reverse[num_train:] y_train = np.load(FileIO(os.path.join(FLAGS.buckets, "20news/Ytrain.npy"), mode='r+')) y_train = to_categorical(y_train) y_test = np.load(FileIO(os.path.join(FLAGS.buckets, "20news/Ytest.npy"), mode='r+')) y_test = to_categorical(y_test) return x_train_0, x_train_1, y_train, x_test_0, x_test_1, y_test, embedding_matrix
def f1(): try: file_path = root_dir + 'data/github/test_codevecs_npy/use.codevecs_0.npy' with FileIO(file_path, mode="rb") as fio: code_vec = np.load(fio) print('读取.npy成功', len(code_vec)) print(code_vec[0][:10]) except Exception as e: print('读取.npy失败') print(e) try: file_path = root_dir + 'data/github/vocab.apiseq.pkl' with FileIO(file_path, mode="rb") as fio: api_seq_vocab = pk.load(fio) print('读取.pkl成功', len(api_seq_vocab)) print(list(api_seq_vocab.keys())[:3]) except Exception as e: print('读取.pkl失败') print(e) try: file_path = root_dir + 'data/github/use.search.txt' with FileIO(file_path, mode="r") as fio: lines = fio.readlines() print('读取.txt成功') print(lines[0]) print(lines[1]) print(lines[2]) except Exception as e: print('读取.txt失败') print(e)
def update_datasets(self, filter=None): if filter is None: filter = self._filter file_list = [] log.info("Updateing datasets from file list: %s", self._source_file) if self._source_file.startswith("gs://"): log.info("Using tensorflow for IO") from tensorflow.python.lib.io.file_io import FileIO input_file = FileIO(self._source_file, "r") log.info("Tensorflow reported size: %d", input_file.size()) else: input_file = open(self._source_file) lines = input_file.readlines() for line in lines: fpath = line.strip() parts = fpath.split("/") file_name = parts[-1] directory_name = "/".join(parts[:-1]) match = self._re.match(file_name) if not match: continue match_components = match.groupdict() dataset_path = self._prepend_path + fpath dataset_id = self.update_dataset(match_components=match_components, dataset_path=dataset_path) dataset = self.get_dataset_by_id(dataset_id) if not filter(dataset_id, match_components, dataset): self.remove_dataset_by_id(dataset_id) input_file.close()
def get_input(): with FileIO(os.path.join(FLAGS.buckets, 'imdb/texts.pkl'), 'r+') as f: texts = pickle.load(f) word_index = pickle.load( FileIO(os.path.join(FLAGS.buckets, "word_index.pkl"), mode='r+')) ngram_index = pickle.load( FileIO(os.path.join(FLAGS.buckets, "ngram_index.pkl"), mode='r+')) sequence = [] for sentence in texts: t_s = [] for token in sentence.split(' '): if token in word_index: t_s.append(str(word_index[token])) sequence.append(t_s) new_sequence = [] for seq in sequence: t_s = [] for i in range(len(seq) - 2): s = '_'.join(seq[i:i + 3]) if s in ngram_index and ngram_index[s] <= num_ngram: t_s.append(ngram_index[s]) new_sequence.append(t_s) new_sequence = pad_sequences(new_sequence, maxlen=max_len) x_train = new_sequence[:25000] x_test = new_sequence[25000:] y_train = np.zeros((25000, ), dtype=np.float32) y_test = np.zeros((25000, ), dtype=np.float32) y_train[12500:25000] = np.ones((12500, ), dtype=np.float32) y_test[12500:25000] = np.ones((12500, ), dtype=np.float32) return x_train, y_train, x_test, y_test
def f3(): a = np.array([5, 4, 3, 2, 1]) file_path = root_dir + 'a.npy' with FileIO(file_path, mode="wb") as fio: np.save(fio, a) with FileIO(file_path, mode="rb") as fio: code_vec = np.load(fio) print('读取.npy成功', code_vec)
def get_input(): with FileIO(os.path.join(FLAGS.buckets, "imdb/texts.pkl"), mode='r+') as f: texts = pickle.load(f) tokenizer = Tokenizer(num_words=num_words) tokenizer.fit_on_texts(texts[0:25000]) sequences = tokenizer.texts_to_sequences(texts) sequences_reverse = [list(reversed(seq)) for seq in sequences] x = pad_sequences(sequences, maxlen=max_len) x_reverse = pad_sequences(sequences_reverse, maxlen=max_len) word_index = tokenizer.word_index embeddings_index = {} wordX = np.load( FileIO(os.path.join(FLAGS.buckets, "glove/embedding.300d.npy"), mode='r+')) allwords = pickle.load( FileIO(os.path.join(FLAGS.buckets, "glove/words.pkl"), mode='r+')) print(len(allwords)) for i in range(len(allwords)): embeddings_index[allwords[i]] = wordX[i, :] embedding_matrix = np.zeros((num_words, 300)) for word, i in word_index.items(): embedding_vector = embeddings_index.get(word) if embedding_vector is not None and i < num_words: embedding_matrix[i] = embedding_vector x_train_0 = x[:25000] x_train_1 = x_reverse[:25000] x_test_0 = x[25000:] x_test_1 = x_reverse[25000:] y_train = np.zeros((25000, ), dtype=np.float32) y_test = np.zeros((25000, ), dtype=np.float32) y_train[12500:25000] = np.ones((12500, ), dtype=np.float32) y_test[12500:25000] = np.ones((12500, ), dtype=np.float32) indice = np.arange(25000) np.random.shuffle(indice) x_train_0 = x_train_0[indice] x_test_0 = x_test_0[indice] x_train_1 = x_train_1[indice] x_test_1 = x_test_1[indice] y_train = y_train[indice] y_test = y_test[indice] result = [] result.append(x_train_0) result.append(x_train_1) result.append(x_test_0) result.append(x_test_1) result.append(y_train) result.append(y_test) result.append(embedding_matrix) return result
def is_database_created(username): filename = "{}.csv".format(username) file_exists = exists_in_gcp(filename) if file_exists: with FileIO(os.path.join("gs://", BUCKET_NAME, filename), 'r') as f: DATABASES[username] = pd.read_csv(f) else: DATABASES[username] = pd.DataFrame( columns=["username", "date", "cause", "spent"]) with FileIO(os.path.join("gs://", BUCKET_NAME, filename), 'w') as f: DATABASES[username].to_csv(f) return not file_exists
def get_input(): f = FileIO(os.path.join(FLAGS.buckets, "texts.pkl"), mode='r+') texts = pickle.load(f) f.close() tokenizer = Tokenizer(nb_words=num_words) tokenizer.fit_on_texts(texts[0:25000]) # sequences = tokenizer.texts_to_sequences(texts) word_index = tokenizer.word_index sequences = [] for i in range(50000): t = [] tokens = texts[i].lower().split(' ') for j in range(len(tokens)): index = word_index.get(tokens[j], 0) if index < num_words: t.append(index) else: t.append(0) sequences.append(t) print('Found %s unique tokens.' % len(word_index)) data1 = pad_sequences(sequences[0:25000], maxlen=max_len) data2 = pad_sequences(sequences[25000:50000], maxlen=max_len) Ytrain = np.zeros((25000,), dtype=np.float32) Ytest = np.zeros((25000,), dtype=np.float32) Ytrain[12500:25000] = np.ones((12500,), dtype=np.float32) Ytest[12500:25000] = np.ones((12500,), dtype=np.float32) Xtrain = np.zeros((25000, (max_len - 3) * 4), dtype=np.int) Xtest = np.zeros((25000, (max_len - 3) * 4), dtype=np.int) for i in range(25000): for j in range(max_len - 3): Xtrain[i, j * 4] = data1[i, j] Xtrain[i, j * 4 + 1] = data1[i][j + 1] + num_words Xtrain[i, j * 4 + 2] = data1[i][j + 2] + num_words * 2 Xtrain[i, j * 4 + 3] = data1[i][j + 3] + num_words * 3 for i in range(25000): for j in range(max_len - 3): Xtest[i, j * 4] = data2[i, j] Xtest[i, j * 4 + 1] = data2[i][j + 1] + num_words Xtest[i, j * 4 + 2] = data2[i][j + 2] + num_words * 2 Xtest[i, j * 4 + 3] = data2[i][j + 3] + num_words * 3 indice = np.arange(25000) np.random.shuffle(indice) Xtrain = Xtrain[indice] Ytrain = Ytrain[indice] Xtest = Xtest[indice] Ytest = Ytest[indice] return Xtrain, Ytrain, Xtest, Ytest
def prepare_train(): print("prepare training data") f = FileIO(os.path.join(FLAGS.buckets, 'texts.pkl'), 'rb') text1 = pickle.load(f) text1 = text1[:25000] f.close() f = FileIO(os.path.join(FLAGS.buckets, 'texts_unsup.pkl'), 'rb') text2 = pickle.load(f) f.close() texts = text1 + text2 tokenizer = Tokenizer(num_words=vocab_size) tokenizer.filters = '' tokenizer.fit_on_texts(texts) sequence = tokenizer.texts_to_sequences(texts) sequence_pad = pad_sequences(sequence, maxlen=MAX_DOCUMENT_LENGTH + 1, dtype=np.int32, padding='post', truncating='post') seq_len = [] for i in range(len(sequence)): r = len(sequence[i]) if r < MAX_DOCUMENT_LENGTH: seq_len.append(r) else: seq_len.append(MAX_DOCUMENT_LENGTH) x_1 = sequence_pad[:, :-1] y_ = sequence_pad[:, 1:] return x_1, seq_len, y_
def write_tfrecord(fname, dataset, log_every=100, pre_fn=None): """Helper function to convert dataset object into tfrecord file. fname must end with .yml or .yaml. The data will be written in a .tfr file with the same suffix. Args: dataset (Dataset): input dataset. fname (str): filename of the dataset to be saved. """ def _bytes_feature(value): """Returns a bytes_list from a string / byte.""" return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) # Preperation tfr = '.'.join(fname.split('.')[:-1] + ['tfr']) writer = tf.python_io.TFRecordWriter(tfr) tensors = dataset.make_one_shot_iterator().get_next() if pre_fn: tensors = pre_fn(tensors) dataset = dataset.map(pre_fn) types = dataset.output_types shapes = dataset.output_shapes # Sanity check assert (type(types) == dict and all(type(v) != dict for v in types.values())),\ "Only dataset of non-nested dictionary is supported." assert fname.endswith('.yml'), "Filename must end with .yml." serialized = {k: tf.serialize_tensor(v) for k, v in tensors.items()} sess = tf.Session() # Writing Loop n_parsed = 0 try: while True: features = {} example = tf.train.Example(features=tf.train.Features( feature={ key: _bytes_feature(val) for key, val in sess.run(serialized).items() })) writer.write(example.SerializeToString()) n_parsed += 1 if n_parsed % log_every == 0: sys.stdout.write('\r {} samples written to {} ...'.format( n_parsed, tfr)) sys.stdout.flush() except tf.errors.OutOfRangeError: print('\r {} samples written to {}, done.'.format(n_parsed, tfr)) sess.close() writer.close() # Write metadata format_dict = { k: { 'dtype': types[k].name, 'shape': shapes[k].as_list() } for k in types.keys() } info_dict = {'n_sample': n_parsed} with FileIO(fname, 'w') as f: yaml.safe_dump({'format': format_dict, 'info': info_dict}, f)
def load_embeddings(vocab, dim, filename): """ Load a subset of embedding vectors from file corresponding to vocabulary provided. Args: vocab: string->int map from words to their ids (id corresponds to vector's row in the resulting embedding matrix). All ids > 0. dim: embedding vector dimension filename: file where each line is a word followed by `dim` floats, all space-separated Returns: MxN = (len(vocab)+1) x dim numpy embedding matrix. The +1 for M is because 0th vector is a zero vector for padding. """ em = np.zeros((len(vocab) + 1, dim), dtype="float32") # with FileIO(filename, "r", encoding="utf-8") as f: with FileIO(filename, "r") as f: for linenum, line in enumerate(f): line = unidecode(line) idx = line.find(' ') if idx < 0: print("malformed line, no space found: line", linenum) continue word = line[:idx] if word not in vocab: continue i = vocab[word] em[i, :] = np.array(line.strip().split()[1:], dtype="float32") return em
def load_tfrecord(fname): """Load tfrecord dataset. Args: fname (str): filename of the .yml metadata file to be loaded. dtypes (dict): dtype of dataset. """ # dataset with FileIO(fname, 'r') as f: format_dict = (yaml.safe_load(f)['format']) dtypes = {k: format_dict[k]['dtype'] for k in format_dict.keys()} shapes = {k: format_dict[k]['shape'] for k in format_dict.keys()} feature_dict = {k: tf.FixedLenFeature([], tf.string) for k in dtypes} def parser(example): return tf.parse_single_example(example, feature_dict) def converter(tensors): tensors = { k: tf.parse_tensor(v, dtypes[k]) for k, v in tensors.items() } [v.set_shape(shapes[k]) for k, v in tensors.items()] return tensors tfr = '.'.join(fname.split('.')[:-1] + ['tfr']) dataset = tf.data.TFRecordDataset(tfr).map(parser).map(converter) return dataset
def fnn_model(): with FileIO(os.path.join(FLAGS.buckets, "docembed.npy"), 'r+') as f: x = np.load(f) x_train = x[:25000] x_test = x[25000:] y_train = np.zeros((25000, ), dtype=np.float32) y_test = np.zeros((25000, ), dtype=np.float32) y_train[12500:25000] = np.ones((12500, ), dtype=np.float32) y_test[12500:25000] = np.ones((12500, ), dtype=np.float32) indice = np.arange(25000) np.random.shuffle(indice) x_train = x_train[indice] x_test = x_test[indice] y_train = y_train[indice] y_test = y_test[indice] x_place = tf.placeholder(dtype=tf.float32, shape=(None, HIDDEN_SIZE)) y_place = tf.placeholder(dtype=tf.int64, shape=(None, )) # out1 = tf.layers.dense(x_place, 100, activation=None) # out2=tf.nn.relu(out1) out3 = tf.layers.dense(x_place, 2, activation=None) output = tf.nn.softmax(out3) predicted_classes = tf.argmax(output, 1) a = tf.cast(tf.equal(y_place, predicted_classes), tf.float32) accuracy = tf.reduce_mean(a) onehot_labels = tf.one_hot(y_place, 2, 1, 0) loss = tf.losses.mean_squared_error(onehot_labels, output) train_op = tf.train.AdamOptimizer().minimize(loss) # train_op = tf.train.GradientDescentOptimizer(learning_rate=0.5).minimize(loss) sess = tf.Session() init = tf.global_variables_initializer() sess.run(init) for i in range(10000): x, y = get_input(x_train, y_train) _loss, _acc, _ = sess.run([loss, accuracy, train_op], feed_dict={ x_place: x, y_place: y }) if i % 100 == 0: print("iter: %d loss: %f accuracy: %f" % (i, _loss, _acc), ) if i % 500 == 0: sum_acc = 0 sum_loss = 0 for j in range(25): _val_loss, _val_acc = sess.run( [loss, accuracy], feed_dict={ x_place: x_test[1000 * j:1000 * (j + 1)], y_place: y_test[1000 * j:1000 * (j + 1)] }) sum_acc += _val_acc sum_loss += _val_loss print('val acc:', sum_acc / 25, 'val loss: ', _val_loss / 25) sess.close()
def get_input(): f = FileIO(os.path.join(FLAGS.buckets, "texts.pkl"), mode='r+') texts = pickle.load(f) f.close() tokenizer = Tokenizer(nb_words=num_words) tokenizer.fit_on_texts(texts[0:25000]) sequences = tokenizer.texts_to_sequences(texts) # word_index = tokenizer.word_index # sequences = [] # for i in range(50000): # t = [] # tokens = texts[i].lower().split(' ') # for j in range(len(tokens)): # index = word_index.get(tokens[j], 0) # if index < num_words: # t.append(index) # else: # t.append(0) # sequences.append(t) data1 = pad_sequences(sequences[0:25000], maxlen=max_len) data2 = pad_sequences(sequences[25000:50000], maxlen=max_len) Ytrain = np.zeros((25000,), dtype=np.float32) Ytest = np.zeros((25000,), dtype=np.float32) Ytrain[12500:25000] = np.ones((12500,), dtype=np.float32) Ytest[12500:25000] = np.ones((12500,), dtype=np.float32) Xtrain = np.zeros((25000, (max_len - 1) * 2), dtype=np.int) Xtest = np.zeros((25000, (max_len - 1) * 2), dtype=np.int) for i in range(25000): for j in range(max_len - 1): Xtrain[i, j * 2] = data1[i, j] Xtrain[i, j * 2 + 1] = data1[i][j + 1] + num_words for i in range(25000): for j in range(max_len - 1): Xtest[i, j * 2] = data2[i, j] Xtest[i, j * 2 + 1] = data2[i][j + 1] + num_words indice = np.arange(25000) np.random.shuffle(indice) Xtrain = Xtrain[indice] Ytrain = Ytrain[indice] Xtest = Xtest[indice] Ytest = Ytest[indice] return Xtrain, Ytrain, Xtest, Ytest
def save_model(model, file): """ Save model to the given file (potentially Google storage). :param model: model :param file: output file """ print('Saving model to file {}.'.format(file)) temp_file = 'temp_model_{}.h5'.format(randint(0, 100000000)) model.save(temp_file) try: # copy model to google storage with FileIO(temp_file, mode='rb') as input_f: with FileIO(file, mode='wb') as output_f: output_f.write(input_f.read()) finally: remove(temp_file)
def potential_model(params, **kwargs): """Shortcut for generating potential model from paramters When creating the model, a params.yml is automatically created in model_dir containing network_params and model_params. The potential model can also be initiated with the model_dir, in that case, params.yml must locate in model_dir from which all parameters are loaded Args: params(str or dict): parameter dictionary or the model_dir **kwargs: additional options for the estimator, e.g. config """ import os import yaml from tensorflow.python.lib.io.file_io import FileIO from datetime import datetime if isinstance(params, str): model_dir = params assert tf.gfile.Exists('{}/params.yml'.format(model_dir)),\ "Parameters files not found." with FileIO(os.path.join(model_dir, 'params.yml'), 'r') as f: params = yaml.load(f, Loader=yaml.Loader) else: model_dir = params['model_dir'] yaml.Dumper.ignore_aliases = lambda *args: True to_write = yaml.dump(params) params_path = os.path.join(model_dir, 'params.yml') if not tf.gfile.IsDirectory(model_dir): tf.gfile.MakeDirs(model_dir) if tf.gfile.Exists(params_path): original = FileIO(params_path, 'r').read() if original != to_write: tf.gfile.Rename( params_path, params_path + '.' + datetime.now().strftime('%y%m%d%H%M')) FileIO(params_path, 'w').write(to_write) model = tf.estimator.Estimator(model_fn=_potential_model_fn, params=params, model_dir=model_dir, **kwargs) return model
def __init__(self, filename): self.i2t = {} with FileIO(filename, mode="r") as fio: lines = fio.readlines() for line in lines: line = line.strip(' \r\n\t') datas = line.split('\t') self.i2t[int(datas[0])] = datas[1] print('load idx ', len(self.i2t))
def get_input(): f = FileIO(os.path.join(FLAGS.buckets, "imdb/texts.pkl"), mode='r+') texts = pickle.load(f) f.close() tokenizer = Tokenizer(nb_words=num_words) tokenizer.fit_on_texts(texts[0:25000]) sequences = tokenizer.texts_to_sequences(texts) sequences_reverse = [list(reversed(seq)) for seq in sequences] x = pad_sequences(sequences, maxlen=max_len) x_reverse=pad_sequences(sequences_reverse, maxlen=max_len) word_index = tokenizer.word_index embeddings_index = {} wordX = np.load(FileIO(os.path.join(FLAGS.buckets, "glove/embedding.300d.npy"), mode='r+')) allwords = pickle.load(FileIO(os.path.join(FLAGS.buckets, "glove/words.pkl"), mode='r+')) for i in range(len(allwords)): embeddings_index[allwords[i]] = wordX[i, :] embedding_matrix = np.zeros((num_words, 300)) for word, i in word_index.items(): embedding_vector = embeddings_index.get(word) if embedding_vector is not None and i < num_words: embedding_matrix[i] = embedding_vector y_train = np.zeros((25000,), dtype=np.float32) y_test = np.zeros((25000,), dtype=np.float32) y_train[12500:25000] = np.ones((12500,), dtype=np.float32) y_test[12500:25000] = np.ones((12500,), dtype=np.float32) x_seq= np.zeros((50000, (max_len - 3) * 4), dtype=np.int) for i in range(50000): for j in range(max_len - 3): x_seq[i, j * 4] = x[i, j] x_seq[i, j * 4 + 1] = x[i][j + 1] + num_words x_seq[i, j * 4 + 2] = x[i][j + 2] + num_words * 2 x_seq[i, j * 4 + 3] = x[i][j + 3] + num_words * 3 x_train_0 = x[:25000] x_train_1 = x_reverse[:25000] x_train_2=x_seq[:25000] x_test_0 = x[25000:] x_test_1 = x_reverse[25000:] x_test_2=x_seq[25000:] result=[] indice = np.arange(25000) np.random.shuffle(indice) result.append(x_train_0[indice]) result.append(x_train_1[indice]) result.append(x_train_2[indice]) result.append(x_test_0[indice]) result.append(x_test_1[indice]) result.append(x_test_2[indice]) result.append(y_train[indice]) result.append(y_test[indice]) result.append(embedding_matrix) return result
def update_datasets(self, filter=None): if filter is None: filter = self._filter close_file = True log.info("Updateing datasets from file list: %s", self._input_source) if hasattr(self._input_source, 'read'): input_file = self._input_source close_file = False elif isinstance(self._input_source, str) and self._input_source.startswith("gs://"): log.info("Using tensorflow for IO") from tensorflow.python.lib.io.file_io import FileIO input_file = FileIO(self._input_source, "r") log.info("Tensorflow reported size: %d", input_file.size()) else: input_file = open(self._input_source) lines = input_file.readlines() for line in lines: fpath = line.strip() parts = fpath.split("/") file_name = parts[-1] match = self._re.match(file_name) if not match: continue match_components = match.groupdict() dataset_path = self._prepend_path + fpath dataset_id = self.update_dataset(match_components=match_components, dataset_path=dataset_path) dataset = self.get_dataset_by_id(dataset_id) if not filter(dataset_id, match_components, dataset): self.remove_dataset_by_id(dataset_id) if close_file: input_file.close()
def update_datasets(self, filter=None): if filter is None: filter = self._filter file_list = [] log.info("Updateing datasets from file list: %s", self._source_file) if self._source_file.startswith("gs://"): log.info("Using tensorflow for IO") from tensorflow.python.lib.io.file_io import FileIO input_file = FileIO(self._source_file, "r") log.info("Tensorflow reported size: %d", input_file.size()) else: input_file = open(self._source_file) lines = input_file.readlines() for line in lines: fpath = line.strip() parts = fpath.split("/") file_name = parts[-1] directory_name = "/".join(parts[:-1]) match = self._re.match(file_name) if not match: continue match_components = match.groupdict() dataset_path = self._prepend_path + fpath dataset_id = self.update_dataset(match_components=match_components, dataset_path=dataset_path) dataset = self.get_dataset_by_id(dataset_id) if not filter(dataset_id, match_components, dataset): self.remove_dataset_by_id(dataset_id) input_file.close()
def save(self): model_json = self.model.to_json() with FileIO("{}/{}.json".format(self.output_path, self.name), "w") as json_file: json_file.write(model_json) fp = "{}.h5".format(self.name) if self.output_path.startswith('gs://'): self.model.save_weights(fp) copy_file_to_gcs(self.output_path, fp) else: self.model.save_weights("{}/{}.h5".format(self.output_path, self.name))
def write_words(word_model, output_file): """Writes the words from a .vec file to an output file of strings. Parameters ---------- word_model : str path to word model file output_file : str path to output file Returns ------- None """ from tensorflow.python.lib.io.file_io import FileIO with FileIO(word_model, 'r') as input_vectors, FileIO(output_file, 'w') as output: for line in input_vectors: split = line.split() if len(split) > 2: word = split[0] output.write(word) output.write("\n")
def load(self, name=""): output_name = self.name if name == "" else name with FileIO("{}/{}.json".format(self.output_path, output_name), "r") as json_file: loaded_model_json = json_file.read() self.model = model_from_json(loaded_model_json) fp = "{}.h5".format(output_name) if self.output_path.startswith('gs://'): copy_file_from_gcs(self.output_path, fp) self.model.load_weights(fp) else: self.model.load_weights("{}/{}.h5".format(self.output_path, output_name))
def copy(source, dest): """ Copy from source to dest, create all necessary dirs. :param source: source file :param dest: dest file """ with FileIO(source, mode='rb') as input_f: if '/' in dest and not isdir(dirname(dest)): makedirs(dirname(dest)) with open(dest, mode='wb') as output_f: while 1: buf = input_f.read(1024 * 1024) if not buf: break output_f.write(buf)
def write_predictions(output_file, tuple_predictions, keys, ground_truth): """ Write predictions to a TSV file. :param output_file: output file :param tuple_predictions: predictions stored in a tuple per sample :param keys: iterable of sample keys (UUIDs) :param ground_truth: ground-truth object that knows the index->label conversion """ if exists(output_file): print('WARNING: Overwriting {}'.format(output_file)) with FileIO(output_file, mode='w') as f: # convert indices to label names using index_to_label for key, indices in zip(keys, tuple_predictions): line = key + ground_truth.to_labels(indices) f.write(line + '\n')
def get_model(): ngram_embed = np.load( FileIO(os.path.join(FLAGS.buckets, "ngram_embedding.npy"), mode='r+')) ngram_embedding = np.random.randn(num_ngram + 1, word_dimension) ngram_embedding[1:] = ngram_embed input_1 = Input(shape=(max_len, )) embedding_1 = Embedding(input_dim=num_ngram + 1, output_dim=word_dimension, weights=[ngram_embedding], trainable=True)(input_1) x = GRU(word_dimension)(embedding_1) # x=Bidirectional(GRU(word_dimension),merge_mode='concat')(embedding_1) output_1 = Dense(1, activation='sigmoid')(x) model = Model(inputs=[input_1], outputs=[output_1]) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def load_data(path, vocab, pad=32, numfiles=0, lowercase=False): X, Xu = [], [] t2s = Text2Seq(vocab, vocab_is_lowercase=lowercase) files = recursively_list_files(path) for i, fname in enumerate(tqdm(files, ascii=True, mininterval=0.5)): if 0 < numfiles < (i + 1): break # Process at most `numfiles` files # with FileIO(fname, "r", encoding="utf-8") as f: with FileIO(fname, "r") as f: text = f.read() seq, aux = t2s.toseq(text) X.extend(seq) Xu.extend(aux) X.extend([0] * pad) Xu.extend([[0, 0]] * pad) X = np.array(X, dtype="int32") Xu = np.array(Xu, dtype="float32") return X, Xu
def load_data_sequences(path, vocab, seqlen, stride, numfiles=0): XX, YY, XXu, YYu = [], [], [], [] t2s = Text2Seq(vocab) files = recursively_list_files(path) for i, fname in enumerate(tqdm(files, ascii=True)): if 0 < numfiles < (i + 1): break # Process at most `numfiles` files with FileIO(fname, "r") as f: seq, unk = t2s.toseq(f.read()) Xi, Yi = seqwindows(seq, seqlen, stride) Xui, Yui = seqwindows(unk, seqlen, stride, dtype="float32") XX.append(Xi) YY.append(Yi) XXu.append(Xui) YYu.append(Yui) X = np.concatenate(XX) Y = np.concatenate(YY) Xu = np.concatenate(XXu) Yu = np.concatenate(YYu) return X, Y, Xu, Yu
def train(): x_place = tf.placeholder(dtype=tf.int64, shape=(batch_size, 1)) y_place = tf.placeholder(dtype=tf.int64, shape=(batch_size, 1)) with tf.device("/cpu:0"): embedding_doc = tf.Variable( tf.random_uniform([num_ngram, 300], -0.5, 0.5)) nce_weights = tf.get_variable('nce_weights_words', [num_words, 300], trainable=True) nce_biases = tf.Variable(tf.zeros([num_words]), trainable=True) input_1 = tf.nn.embedding_lookup(embedding_doc, x_place) input_2 = tf.reshape(input_1, [-1, 300]) loss = tf.reduce_mean( tf.nn.nce_loss(weights=nce_weights, biases=nce_biases, labels=y_place, inputs=input_2, num_sampled=num_sampled, num_classes=num_words)) optimizer = tf.train.AdamOptimizer().minimize(loss) init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) x, y, embedding_metrix = prepare_data() init_nce = tf.assign(nce_weights, embedding_metrix) sess.run(init_nce) start = 0 for i in range(1000000): x_1, _y, start = get_input(x, y, start) # _loss, _ = sess.run([loss, optimizer], feed_dict={x1_place: x_1, x2_place: x_2, y_place: _y}) _loss, _ = sess.run([loss, optimizer], feed_dict={ x_place: x_1, y_place: _y }) if i % 300 == 0: print(i, " loss ", _loss) np.save( FileIO(os.path.join(FLAGS.buckets, "ngram_embedding.npy"), mode='w+'), embedding_doc.eval(sess))
def load_vocab(filename, maxwords=0): """ Load newline-separated words from file to dict mapping them to unique ids. :param maxwords: Max number of words to load. Load all by default. Returns (list of words, word->id map) """ pad = "·" # "<#PAD#>" vocab = dict() words = [] counter = 1 # start off with 1 so that embedding matrix's first vector is zero and second is for unknown words.append(pad) with FileIO(filename, "r") as f: for i, line in enumerate(f): if 0 < maxwords < i + 1: break word = line.strip() words.append(word) vocab[word] = counter counter += 1 return words, vocab