def rollout_to_protobuf(document_words, document_tags): print("document_words") print(document_words) print("document_tags") print(document_tags) input_ids, input_masks, y_masks, text, ys = tokenize([document_words], [document_tags]) print("text") print(text) example_proto = Example(features=Features( feature={ 'text': Feature(bytes_list=tf.train.BytesList( value=[w.encode('utf-8') for w in text[0]])), 'input_ids': Feature(int64_list=tf.train.Int64List(value=input_ids)), 'input_masks': Feature(int64_list=tf.train.Int64List(value=input_masks)), 'y_masks': Feature(int64_list=tf.train.Int64List(value=y_masks)), 'labels': Feature(int64_list=tf.train.Int64List(value=ys)) })) writer.write(example_proto.SerializeToString())
def serialise(data): ID,pos, dimensions, color, border, fill, text, img, seq_len, seq_mask = data['ID'], data['pos'], data['dimensions'], data['color'], \ data['border'], data['fill'], data['text'], data['img'], \ int(data['seq_len']), data['seq_mask'] \ ID = Feature(bytes_list=BytesList(value=[tf.io.serialize_tensor(ID).numpy(),])) pos = Feature(bytes_list=BytesList(value=[tf.io.serialize_tensor(tf.cast(pos, tf.float32)).numpy(),])) dimensions = Feature(bytes_list=BytesList(value=[tf.io.serialize_tensor(tf.cast(dimensions, tf.float32)).numpy(),])) color = Feature(bytes_list=BytesList(value=[tf.io.serialize_tensor(tf.cast(color, tf.float32)).numpy(),])) border = Feature(bytes_list=BytesList(value=[tf.io.serialize_tensor(tf.cast(border, tf.float32)).numpy(),])) fill = Feature(bytes_list=BytesList(value=[tf.io.serialize_tensor(tf.cast(fill, tf.float32)).numpy(),])) text = Feature(bytes_list=BytesList(value=[tf.io.serialize_tensor(tf.cast(text, tf.float32)).numpy(),])) img = Feature(bytes_list=BytesList(value=[img.numpy(),])) seq_len = Feature(int64_list=Int64List(value=[seq_len,])) seq_mask = Feature(bytes_list=BytesList(value=[tf.io.serialize_tensor(seq_mask).numpy(),])) # img is already serialised because we never decode it! features = Features(feature={ 'ID' : ID, 'pos' : pos, 'dimensions' : dimensions, 'color' : color, 'border' : border, 'fill' : fill, 'text' : text, 'img': img, 'seq_len':seq_len, 'seq_mask':seq_mask, }) example = Example(features=features) return example.SerializeToString()
def serialise_traj(data): features = {k: Feature(bytes_list=BytesList(value=[tf.io.serialize_tensor(v).numpy(),])) for k,v in data.items() if k not in ['seq_lens']} features['seq_lens'] = Feature(int64_list=Int64List(value=[data['seq_lens'],])) example = Example(features=Features(feature=features)) return example.SerializeToString()
def _serialize_features(self, input_ids, input_mask, segment_ids, label_ids): features = { "input_ids": self._create_int_feature(input_ids), "input_mask": self._create_int_feature(input_mask), "segment_ids": self._create_int_feature(segment_ids), "label_ids": self._create_int_feature(label_ids) } tf_example = Example(features=Features(feature=features)) return tf_example.SerializeToString()
def serialise_vid(data): # seq_lens, masks, imgs, goal_imgs,label, label_embedding, tag = data['seq_lens'], data['masks'], data['imgs'], data['goal_imgs'], data['label'], data['label_embedding'], data['tag'] features = {k: Feature(bytes_list=BytesList(value=[tf.io.serialize_tensor(v).numpy(),])) for k,v in data.items() if k not in ['seq_lens']} features['seq_lens'] = Feature(int64_list=Int64List(value=[data['seq_lens'],])) example = Example(features=Features(features)) return example.SerializeToString()
def __encode_input(self, mr, input_encoder): """Encodes the input, and creates a TF Example record out of it.""" input_ids = input_encoder.encode(mr) input_ids.append(text_encoder.EOS_ID) features = {'inputs': Feature(int64_list=Int64List(value=input_ids))} example = Example(features=Features(feature=features)) return example.SerializeToString()
def save(mask, img, save_dir, count=0): mask = tf.io.encode_jpeg(mask) img = tf.io.encode_jpeg(img) Image_Buffer = Example(features=Features( feature={ "Image": Feature(bytes_list=BytesList(value=[img.numpy()])), "Mask": Feature(bytes_list=BytesList(value=[mask.numpy()])) })) with tf.io.TFRecordWriter(os.path.join(save_dir, f"Data-{count}.tfrecord")) as f: f.write(Image_Buffer.SerializeToString())
def create_examples(data, bert_client, training=True, label2int=None, class_weight=None): """ data: pd.DataFrame label2int: dict class_weight: list yield examples """ idx_start = data.index[0] A_encoded = bert_client.encode(data['title1_en'].tolist()) B_encoded = bert_client.encode(data['title2_en'].tolist()) for i in range(len(data)): feature = { 'A_encoded': Feature(float_list=FloatList(value=A_encoded[i])), 'B_encoded': Feature(float_list=FloatList(value=B_encoded[i])) } if training: label = label2int[data.loc[idx_start + i, 'label']] feature['label'] = Feature(int64_list=Int64List(value=[label])) feature['class_weight'] = Feature(float_list=FloatList( value=[class_weight[label]])) else: feature['id'] = Feature(int64_list=Int64List( value=[data.loc[idx_start + i, 'id']])) yield Example(features=Features(feature=feature))
def generator(): """ Генератор данных. Считывает данные из папки и токенезирует их Yield: dict: 'article_text' - текст 'extended_article_tokens' - tf.Tensor: последовательность токенов текста, закодированных с использование oov слов 'summary_text' - резюме 'extended_summary_tokens' - tf.Tensor: последовательность токенов резюме, закодированных с использование oov слов 'oovs' - словарь id2word для oov слов пары текст-резюме """ np.random.shuffle(all_files) for file in tqdm(all_files): reader = open(file, 'rb') while True: len_bytes = reader.read(8) if not len_bytes: break str_len = struct.unpack('q', len_bytes)[0] example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0] tf_example = Example.FromString(example_str) article, summary = self.extract_text_data(tf_example) extended_tokenized_article, extended_tokenized_summary, oovs = text_tokenizer(article, summary, max_article_len, max_summary_len) extended_tokenized_article = tf.constant(extended_tokenized_article) extended_tokenized_summary = tf.constant(extended_tokenized_summary) yield {'article_text': article, 'extended_article_tokens': extended_tokenized_article, 'summary_text': summary, 'extended_summary_tokens': extended_tokenized_summary, 'oovs': oovs}
def make_tf_examples(string_features, int_features, labels): int_features += [[label] for label in zero_norm_labels(labels)] string_features = [ Feature(bytes_list=BytesList(value=val)) for val in string_features ] int_features = [ Feature(int64_list=Int64List(value=val)) for val in int_features ] all_features = string_features + int_features return [ Example(features=Features( feature={ "left": left, "target": target, "right": right, "left_ids": left_ids, "target_ids": target_ids, "right_ids": right_ids, "labels": label, })) for ( left, target, right, left_ids, target_ids, right_ids, label, ) in zip(*split_list(all_features, parts=7)) ]
def convert_to_example( adj, feature, label_data=None, label_mask=None, ): """ Writes graph related data to disk. """ adj_row, adj_col = np.nonzero(adj) adj_values = adj[adj_row, adj_col] adj_elem_len = len(adj_row) degrees = np.sum(adj, 0) adj_degrees = [] for ar, ac in zip(adj_row, adj_col): if ar == ac: adj_degrees.append(0) else: adj_degrees.append(int(degrees[ar])) feature = np.array(feature) feature_row, feature_col = np.nonzero(feature) feature_values = feature[feature_row, feature_col] feature_elem_len = len(feature_row) feature = { 'adj_row': Feature(int64_list=Int64List(value=list(adj_row))), 'adj_column': Feature(int64_list=Int64List(value=list(adj_col))), 'adj_values': Feature(float_list=FloatList(value=list(adj_values))), 'adj_elem_len': Feature(int64_list=Int64List(value=[adj_elem_len])), 'adj_degrees': Feature(int64_list=Int64List(value=adj_degrees)), 'feature_row': Feature(int64_list=Int64List(value=list(feature_row))), 'feature_column': Feature(int64_list=Int64List(value=list(feature_col))), 'feature_values': Feature(float_list=FloatList(value=list(feature_values))), 'feature_elem_len': Feature(int64_list=Int64List(value=[feature_elem_len])), 'size': Feature(int64_list=Int64List(value=list(feature.shape))) } if label_data is not None: label_data = np.nan_to_num(label_data) feature['label'] = Feature(int64_list=Int64List( value=label_data.astype(int))) feature['mask_label'] = Feature(int64_list=Int64List( value=label_mask.astype(int))), features = Features(feature=feature) ex = Example(features=features) return ex.SerializeToString()
def create_example(features: np.ndarray, label: np.int32): return Example(features=Features( feature={ "features": NumpyToRecordConverter._bytes_feature( tf.io.serialize_tensor(features)), "label": Feature(int64_list=Int64List(value=[label])) })).SerializeToString()
def write_to_tfrecords(adj, feature, label_data, label_mask, tfrname): """ Writes graph related data to disk. """ adj_row, adj_col = np.nonzero(adj) adj_values = adj[adj_row, adj_col] adj_elem_len = len(adj_row) feature = np.array(feature) feature_row, feature_col = np.nonzero(feature) feature_values = feature[feature_row, feature_col] feature_elem_len = len(feature_row) features = Features( feature={ 'label': Feature(int64_list=Int64List(value=label_data)), 'mask_label': Feature(int64_list=Int64List(value=label_mask)), 'adj_row': Feature(int64_list=Int64List(value=list(adj_row))), 'adj_column': Feature(int64_list=Int64List(value=list(adj_col))), 'adj_values': Feature(float_list=FloatList(value=list(adj_values))), 'adj_elem_len': Feature(int64_list=Int64List(value=[adj_elem_len])), 'feature_row': Feature(int64_list=Int64List(value=list(feature_row))), 'feature_column': Feature(int64_list=Int64List(value=list(feature_col))), 'feature_values': Feature(float_list=FloatList(value=list(feature_values))), 'feature_elem_len': Feature(int64_list=Int64List(value=[feature_elem_len])), 'size': Feature(int64_list=Int64List(value=list(feature.shape))) }) ex = Example(features=features) with TFRecordWriter(tfrname) as single_writer: single_writer.write(ex.SerializeToString())
def get_cycle_example(cell_value, summary_idx, cycle_idx, scaling_factors): """ Define the columns that should be written to tfrecords and converts the raw data to "Example" objects. Every Example contains data from one charging cycle. The data is scaled (divided) by the corresponding values in "scaling_factors". """ # Summary feature values (scalars --> have to be wrapped in lists) ir_value = [ cell_value["summary"][cst.INTERNAL_RESISTANCE_NAME][summary_idx] / scaling_factors[cst.INTERNAL_RESISTANCE_NAME] ] qd_value = [ cell_value["summary"][cst.QD_NAME][summary_idx] / scaling_factors[cst.QD_NAME] ] rc_value = [ cell_value["summary"][cst.REMAINING_CYCLES_NAME][summary_idx] / scaling_factors[cst.REMAINING_CYCLES_NAME] ] dt_value = [ cell_value["summary"][cst.DISCHARGE_TIME_NAME][summary_idx] / scaling_factors[cst.DISCHARGE_TIME_NAME] ] cc_value = [float(cycle_idx) / scaling_factors[cst.REMAINING_CYCLES_NAME] ] # Same scale --> same scaling factor # Detail feature values (arrays) qdlin_value = cell_value["cycles"][cycle_idx][ cst.QDLIN_NAME] / scaling_factors[cst.QDLIN_NAME] tdlin_value = cell_value["cycles"][cycle_idx][ cst.TDLIN_NAME] / scaling_factors[cst.TDLIN_NAME] # Wrapping as example cycle_example = Example(features=Features( feature={ cst.INTERNAL_RESISTANCE_NAME: Feature(float_list=FloatList(value=ir_value)), cst.QD_NAME: Feature(float_list=FloatList(value=qd_value)), cst.REMAINING_CYCLES_NAME: Feature(float_list=FloatList(value=rc_value)), cst.DISCHARGE_TIME_NAME: Feature(float_list=FloatList(value=dt_value)), cst.QDLIN_NAME: Feature(float_list=FloatList(value=qdlin_value)), cst.TDLIN_NAME: Feature(float_list=FloatList(value=tdlin_value)), cst.CURRENT_CYCLE_NAME: Feature(float_list=FloatList(value=cc_value)) })) return cycle_example
def main(): model, signature, batch_file_path, sentence, target = parse_args() feat_dict = {"sentences": [], "targets": []} if batch_file_path is not None: with open(batch_file_path, "r") as batch_file: fieldnames = ["target", "sentence"] csvreader = DictReader(batch_file, fieldnames=fieldnames) for row in csvreader: feat_dict["targets"].append(row["target"].strip()) feat_dict["sentences"].append(row["sentence"].strip()) else: feat_dict["targets"].append(target) feat_dict["sentences"].append(sentence) l_ctxts, trgs, r_ctxts = FeatureProvider.partition_sentences( sentences=feat_dict["sentences"], targets=feat_dict["targets"], offsets=FeatureProvider.get_target_offset_array(feat_dict), ) l_enc = [ FeatureProvider.tf_encode_tokens(tokens) for tokens in FeatureProvider.tokenize_phrases(l_ctxts) ] trg_enc = [ FeatureProvider.tf_encode_tokens(tokens) for tokens in FeatureProvider.tokenize_phrases(trgs) ] r_enc = [ FeatureProvider.tf_encode_tokens(tokens) for tokens in FeatureProvider.tokenize_phrases(r_ctxts) ] tf_examples = [] for left, target, right in zip(l_enc, trg_enc, r_enc): features = Features( feature={ "left": Feature(bytes_list=BytesList(value=left)), "target": Feature(bytes_list=BytesList(value=target)), "right": Feature(bytes_list=BytesList(value=right)), } ) tf_example = Example(features=features) tf_examples.append(tf_example.SerializeToString()) tensor_proto = make_tensor_proto( tf_examples, dtype=tf_string, shape=[len(tf_examples)] ) channel = insecure_channel("127.0.0.1:8500") stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) # CLASSIFICATION classification_req = ClassificationRequest() inputs = Input(example_list=ExampleList(examples=[tf_example])) classification_req.input.CopyFrom(inputs) # pylint: disable=E1101 classification_req.model_spec.name = "lg" # pylint: disable=E1101 classification = stub.Classify(classification_req, 60.0) print(classification) # PREDICTION prediction_req = PredictRequest() prediction_req.inputs["instances"].CopyFrom( # pylint: disable=E1101 tensor_proto ) prediction_req.model_spec.signature_name = ( # pylint: disable=E1101 signature ) prediction_req.model_spec.name = model # pylint: disable=E1101 prediction = stub.Predict(prediction_req, 60.0) print(prediction)
def serialise(data): obs, acts, goals, seq_lens, masks, dataset_path, tstep_idxs , imgs , goal_imgs, proprioceptive_features = data['obs'], \ data['acts'], data['goals'], data['seq_lens'], data['masks'], data['dataset_path'], data['tstep_idxs'], data['imgs'], data['goal_imgs'], data['proprioceptive_features'] # obs (1, 40, 18) # acts (1, 40, 7) # goals (1, 40, 11) # seq_lens (1,) # masks (1, 40) # dataset_path (1, 40) # tstep_idxs (1, 40) # imgs (1, 40, 200, 200, 3) # goal_imgs (1, 40, 200, 200, 3) # proprioceptive_features (1, 40, 7) goal_imgs = tf.expand_dims( goal_imgs[:, 0, :, :, :], 1) # crete a :, 1, :,:,: shaped goal images for less file IO obs = Feature(bytes_list=BytesList(value=[ tf.io.serialize_tensor(tf.squeeze(obs)).numpy(), ])) acts = Feature(bytes_list=BytesList(value=[ tf.io.serialize_tensor(tf.squeeze(acts)).numpy(), ])) goals = Feature(bytes_list=BytesList(value=[ tf.io.serialize_tensor(tf.squeeze(goals)).numpy(), ])) seq_lens = Feature(int64_list=Int64List(value=[ seq_lens, ])) masks = Feature(bytes_list=BytesList(value=[ tf.io.serialize_tensor(tf.squeeze(masks)).numpy(), ])) imgs = Feature(bytes_list=BytesList(value=[ tf.io.serialize_tensor(tf.squeeze(imgs)).numpy(), ])) goal_imgs = Feature(bytes_list=BytesList(value=[ tf.io.serialize_tensor(tf.squeeze(goal_imgs)).numpy(), ])) proprioceptive_features = Feature(bytes_list=BytesList(value=[ tf.io.serialize_tensor(tf.squeeze(proprioceptive_features)).numpy(), ])) features = Features( feature={ 'obs': obs, 'acts': acts, 'goals': goals, 'seq_lens': seq_lens, 'masks': masks, 'imgs': imgs, 'goal_imgs': goal_imgs, 'proprioceptive_features': proprioceptive_features }) example = Example(features=features) return example.SerializeToString() # Sample Usage # r = lfp.data.PlayDataloader(include_imgs = args.images, batch_size=1, window_size=args.window_size_max, min_window_size=args.window_size_min) # rd = r.extract(TRAIN_DATA_PATHS, from_tfrecords=args.from_tfrecords) # rd = r.load(rd) # r_it = iter(rd) # @tf.function # def sample(): # return r_it.next() # data_paths = [str(STORAGE_PATH/'precompute')+f"/{x}.tfrecords" for x in range(0,8)] # #@title write to gcs # from tqdm import tqdm # for path in data_paths: # with tf.io.TFRecordWriter(path) as file_writer: # print(path) # for i in tqdm(range(0,200)): # byte_stream = serialise(sample()) # file_writer.write(byte_stream)