def generator(self): try: for feed_list in super(GraphGenerator, self).generator(): yield feed_list except Exception as e: log.exception(e)
def start(self): batch = 0 start = time.time() trainer_id = int(os.getenv("PADDLE_TRAINER_ID", "0")) if trainer_id == 0: writer = SummaryWriter( os.path.join(self.config.output_path, "train_history")) for epoch_idx in range(self.config.epoch): for idx, batch_feed_dict in enumerate(self.model.data_loader()): try: cpu_time = time.time() batch += 1 batch_loss = self.exe.run(self.program, feed=batch_feed_dict, fetch_list=[self.model.loss]) end = time.time() if trainer_id == 0: writer.add_scalar("loss", np.mean(batch_loss), batch) if batch % self.config.log_per_step == 0: log.info( "Epoch %s Batch %s %s-Loss %s \t Speed(per batch) %.5lf/%.5lf sec" % (epoch_idx, batch, "train", np.mean(batch_loss), (end - start) / batch, (end - cpu_time))) writer.flush() if batch % self.config.save_per_step == 0: self.fleet.save_persistables( self.exe, os.path.join(self.config.output_path, str(batch))) except Exception as e: log.info("Pyreader train error") log.exception(e) log.info("epcoh %s done." % epoch_idx)
def pair_generate(self): for walks in self.walk_generator(): try: src_list, pos_list = [], [] for walk in walks: s, p = skip_gram_gen_pair(walk, self.config.win_size) src_list.append(s), pos_list.append(p) src = [s for x in src_list for s in x] pos = [s for x in pos_list for s in x] if len(src) == 0: continue negs = self.negative_sample( src, pos, neg_num=self.config.neg_num, neg_sample_type=self.config.neg_sample_type) src = np.array(src, dtype=np.int64).reshape(-1, 1, 1) pos = np.array(pos, dtype=np.int64).reshape(-1, 1, 1) yield src, pos, negs except Exception as e: log.exception(e)
def train_prog(exe, program, model, pyreader, args): trainer_id = int(os.getenv("PADDLE_TRAINER_ID", "0")) start = time.time() batch = 0 total_loss = 0. total_acc = 0. total_sample = 0 for epoch_idx in range(args.num_epoch): for step, batch_feed_dict in enumerate(pyreader()): try: cpu_time = time.time() batch += 1 batch_loss, batch_acc = exe.run( program, feed=batch_feed_dict, fetch_list=[model.loss, model.acc]) end = time.time() if batch % args.log_per_step == 0: log.info( "Batch %s Loss %s Acc %s \t Speed(per batch) %.5lf/%.5lf sec" % (batch, np.mean(batch_loss), np.mean(batch_acc), (end - start) / batch, (end - cpu_time))) if step % args.steps_per_save == 0: save_path = args.save_path if trainer_id == 0: model_path = os.path.join(save_path, "%s" % step) fleet.save_persistables(exe, model_path) except Exception as e: log.info("Pyreader train error") log.exception(e)
def generator(self): try: for feed_dict in super(GraphGenerator, self).generator(): if self.use_pyreader: yield [feed_dict[name] for name in self.feed_name_list] else: yield feed_dict except Exception as e: log.exception(e)
def __call__(self): iterval = 20000000 * 24 // self.config.walk_len pair_count = 0 for walks in self.walk_generator(): try: for walk in walks: index = np.arange(0, len(walk), dtype="int64") batch_s, batch_p = skip_gram_gen_pair( index, self.config.win_size) for s, p in zip(batch_s, batch_p): yield walk[s], walk[p] pair_count += 1 if pair_count % iterval == 0 and self.rank == 0: log.info("[%s] pairs have been loaded in rank [%s]" \ % (pair_count, self.rank)) except Exception as e: log.exception(e) log.info("total [%s] pairs in rank [%s]" % (pair_count, self.rank))
def __call__(self): np.random.seed(os.getpid()) if self.neg_sample_type == "outdegree": outdegree = self.graph.outdegree() distribution = 1. * outdegree / outdegree.sum() alias, events = alias_sample_build_table(distribution) max_len = int(self.batch_size * self.walk_len * ((1 + self.win_size) - 0.3)) for walks in self.walk_generator(): try: src_list, pos_list = [], [] for walk in walks: s, p = skip_gram_gen_pair(walk, self.win_size) src_list.append(s[:max_len]), pos_list.append(p[:max_len]) src = [s for x in src_list for s in x] pos = [s for x in pos_list for s in x] src = np.array(src, dtype=np.int64), pos = np.array(pos, dtype=np.int64) src, pos = np.reshape(src, [-1, 1, 1]), np.reshape(pos, [-1, 1, 1]) neg_sample_size = [len(pos), self.neg_num, 1] if src.shape[0] == 0: continue if self.neg_sample_type == "average": negs = np.random.randint(low=0, high=self.graph.num_nodes, size=neg_sample_size) elif self.neg_sample_type == "outdegree": negs = alias_sample(neg_sample_size, alias, events) elif self.neg_sample_type == "inbatch": pass dst = np.concatenate([pos, negs], 1) # [batch_size, 1, 1] [batch_size, neg_num+1, 1] yield src[:max_len], dst[:max_len] except Exception as e: log.exception(e)