def fetch_data(self, fetch_type=1): if fetch_type == NewsDnnBaseDataReader.DictDataTerm["Train"]: self.__train_cursor = self.db.get_data( self.configs['database']['name'], self.configs['database']['train']['query'], self.configs['database']['fields'], notimeout=True) if self.configs['database']['sort'] is not None: self.__train_cursor = self.__train_cursor.sort( ListHelper.convert_dict_list( self.configs['database']['sort'])) elif fetch_type == NewsDnnBaseDataReader.DictDataTerm["Validate"]: self.__validate_cursor = self.db.get_data( self.configs['database']['name'], self.configs['database']['validate']['query'], self.configs['database']['fields'], notimeout=True) if self.configs['database']['sort'] is not None: self.__validate_cursor = self.__validate_cursor.sort( ListHelper.convert_dict_list( self.configs['database']['sort'])) elif fetch_type == NewsDnnBaseDataReader.DictDataTerm["Test"]: self.__test_cursor = self.db.get_data( self.configs['database']['name'], self.configs['database']['test']['query'], self.configs['database']['fields'], notimeout=True) if self.configs['database']['sort'] is not None: self.__test_cursor = self.__test_cursor.sort( ListHelper.convert_dict_list( self.configs['database']['sort'])) else: LoggerHelper.critical('Unable To Fetch')
def get_count(self, fetch_type=1): if fetch_type == NewsDnnBaseDataReader.DictDataTerm["Train"]: if self.__train_cursor is None: self.fetch_data(NewsDnnBaseDataReader.DictDataTerm["Train"]) self.train_count = self.__train_cursor.count() return self.train_count elif fetch_type == NewsDnnBaseDataReader.DictDataTerm["Validate"]: if self.__validate_cursor is None: self.fetch_data(NewsDnnBaseDataReader.DictDataTerm["Validate"]) self.validate_count = self.__validate_cursor.count() return self.validate_count elif fetch_type == NewsDnnBaseDataReader.DictDataTerm["Test"]: if self.__test_cursor is None: self.fetch_data(NewsDnnBaseDataReader.DictDataTerm["Test"]) self.test_count = self.__test_cursor.count() return self.test_count else: LoggerHelper.critical('Unable To Fetch')
def get_data(self, fetch_type=1, data_type=1): if fetch_type == NewsDnnBaseDataReader.DictDataTerm["Train"]: cursor = self.__train_cursor elif fetch_type == NewsDnnBaseDataReader.DictDataTerm["Validate"]: cursor = self.__validate_cursor elif fetch_type == NewsDnnBaseDataReader.DictDataTerm["Test"]: cursor = self.__test_cursor else: LoggerHelper.critical('Unable To Get Cursor (Check Fetch Type)') return None cursor.rewind() self.clear_data() if data_type == NewsDnnBaseDataReader.DictDataType["News"]: return self.get_data_news(cursor) elif data_type == NewsDnnBaseDataReader.DictDataType["Wiki"]: return self.get_data_wiki(cursor) elif data_type == NewsDnnBaseDataReader.DictDataType["WikiAndTweet"]: return self.get_data_wiki_and_tweet(cursor) else: LoggerHelper.critical('Unknown Data Type (data_type)') return None