def _get_evaluation_data(self): jvalue = callZooFunc("float", "createMiniBatchRDDFromTFDatasetEval", self.rdd.map(lambda x: x[0]), self.init_op_name, self.table_init_op, self.output_names, self.output_types, self.shard_index_op_name) rdd = jvalue.value().toJavaRDD() return rdd
def _get_training_data(self): jvalue = callZooFunc("float", "createTFDataFeatureSet", self.rdd.map(lambda x: x[0]), self.init_op_name, self.table_init_op, self.output_names, self.output_types, self.shard_index_op_name, self.inter_threads, self.intra_threads) return FeatureSet(jvalue=jvalue)
def _get_validation_data(self): if self.validation_dataset is not None: jvalue = callZooFunc("float", "createTFDataFeatureSet", self.val_rdd.map(lambda x: x[0]), self.init_op_name, self.table_init_op, self.output_names, self.output_types, self.shard_index_op_name) return FeatureSet(jvalue=jvalue) return None
def _get_evaluation_data(self): feature_length = len(nest.flatten(self.tensor_structure[0])) jvalue = callZooFunc("float", "createMiniBatchRDDFromTFDatasetEval", self.rdd.map(lambda x: x[0]), self.init_op_name, self.table_init_op, self.output_names, self.output_types, self.shard_index_op_name, feature_length) rdd = jvalue.value().toJavaRDD() return rdd
def _get_prediction_data(self): assert not self.drop_remainder, \ "sanity check: drop_remainder should be false in this case," \ " otherwise please report a bug" jvalue = callZooFunc("float", "createMiniBatchRDDFromTFDataset", self.rdd.map(lambda x: x[0]), self.init_op_name, self.table_init_op, self.output_names, self.output_types, self.shard_index_op_name) rdd = jvalue.value().toJavaRDD() return rdd