def loadCorpus(self, dirName): """Load/create the conversations data Args: dirName (str): The directory where to load/save the model """ datasetExist = False if os.path.exists(os.path.join(dirName, self.samplesName)): datasetExist = True if not datasetExist: # First time we load the database: creating all files print('Training samples not found. Creating dataset...') # Corpus creation if self.args.corpus == 'cornell': cornellData = CornellData(self.corpusDir) self.createCorpus(cornellData.getConversations()) elif self.args.corpus == 'opensubs': opensubsData = OpensubsData(self.corpusDir) self.createCorpus(opensubsData.getConversations()) elif self.args.corpus == 'fbdata': fbData = FBData(self.corpusDir) self.createCorpus(fbData.getConversations()) # Saving print('Saving dataset...') self.saveDataset(dirName) # Saving tf samples else: print('Loading dataset from {}...'.format(dirName)) self.loadDataset(dirName) assert self.padToken == 0
def loadCorpus(self, dirName): """Load/create the conversations data Args: dirName (str): The directory where to load/save the model """ datasetExist = False if os.path.exists(os.path.join(dirName, self.samplesName)): datasetExist = True if not datasetExist: # First time we load the database: creating all files print('Training samples not found. Creating dataset...') # Corpus creation if self.args.corpus == 'cornell': cornellData = CornellData(self.corpusDir) self.createCorpus(cornellData.getConversations()) elif self.args.corpus == 'nutrition': mealData = MealData(self.corpusDir) if self.args.encode_food_descrips: self.createCorpus( zip(mealData.getFoodDescrips(), mealData.getMeals())) elif self.args.encode_single_food_descrip: self.createCorpus( zip(mealData.getSingleFoodDescrips(), mealData.getAlignments())) elif self.args.encode_food_ids: self.createCorpus( zip(mealData.getFoodIDs(), mealData.getMeals())) else: self.createCorpus(mealData.getMeals()) elif self.args.corpus == 'healthy-comments': self.healthyData = HealthyData( self.corpusDir, self.args.usda_vecs, self.args.healthy_flag, self.args.augment, self.args.motivate_only, self.args.advice_only, self.args.all_data) if self.args.encode_food_ids: self.createCorpus( zip(self.healthyData.getFoodIDs(), self.healthyData.getResponses())) else: self.createCorpus( zip(self.healthyData.getMeals(), self.healthyData.getResponses(), self.healthyData.getFoodEmb())) # Saving print('Saving dataset...') self.saveDataset(dirName) # Saving tf samples else: print('Loading dataset from {}...'.format(dirName)) self.loadDataset(dirName) if self.args.finetune and not self.args.test: self.trainingSamples = [] mealData = MealData('/usr/users/zcollins/Data_Files/allfood/') self.createCorpus(mealData.getMeals()) assert self.padToken == 0