def fetch_data(self, fetch_type=1):
if fetch_type == NewsDnnBaseDataReader.DictDataTerm["Train"]:
self.__train_cursor = self.db.get_data(
self.configs['database']['name'],
self.configs['database']['train']['query'],
self.configs['database']['fields'],
notimeout=True)
if self.configs['database']['sort'] is not None:
self.__train_cursor = self.__train_cursor.sort(
ListHelper.convert_dict_list(
self.configs['database']['sort']))
elif fetch_type == NewsDnnBaseDataReader.DictDataTerm["Validate"]:
self.__validate_cursor = self.db.get_data(
self.configs['database']['name'],
self.configs['database']['validate']['query'],
self.configs['database']['fields'],
notimeout=True)
if self.configs['database']['sort'] is not None:
self.__validate_cursor = self.__validate_cursor.sort(
ListHelper.convert_dict_list(
self.configs['database']['sort']))
elif fetch_type == NewsDnnBaseDataReader.DictDataTerm["Test"]:
self.__test_cursor = self.db.get_data(
self.configs['database']['name'],
self.configs['database']['test']['query'],
self.configs['database']['fields'],
notimeout=True)
if self.configs['database']['sort'] is not None:
self.__test_cursor = self.__test_cursor.sort(
ListHelper.convert_dict_list(
self.configs['database']['sort']))
else:
LoggerHelper.critical('Unable To Fetch')
def get_count(self, fetch_type=1):
if fetch_type == NewsDnnBaseDataReader.DictDataTerm["Train"]:
if self.__train_cursor is None:
self.fetch_data(NewsDnnBaseDataReader.DictDataTerm["Train"])
self.train_count = self.__train_cursor.count()
return self.train_count
elif fetch_type == NewsDnnBaseDataReader.DictDataTerm["Validate"]:
if self.__validate_cursor is None:
self.fetch_data(NewsDnnBaseDataReader.DictDataTerm["Validate"])
self.validate_count = self.__validate_cursor.count()
return self.validate_count
elif fetch_type == NewsDnnBaseDataReader.DictDataTerm["Test"]:
if self.__test_cursor is None:
self.fetch_data(NewsDnnBaseDataReader.DictDataTerm["Test"])
self.test_count = self.__test_cursor.count()
return self.test_count
else:
LoggerHelper.critical('Unable To Fetch')
def get_data(self, fetch_type=1, data_type=1):
if fetch_type == NewsDnnBaseDataReader.DictDataTerm["Train"]:
cursor = self.__train_cursor
elif fetch_type == NewsDnnBaseDataReader.DictDataTerm["Validate"]:
cursor = self.__validate_cursor
elif fetch_type == NewsDnnBaseDataReader.DictDataTerm["Test"]:
cursor = self.__test_cursor
else:
LoggerHelper.critical('Unable To Get Cursor (Check Fetch Type)')
return None
cursor.rewind()
self.clear_data()
if data_type == NewsDnnBaseDataReader.DictDataType["News"]:
return self.get_data_news(cursor)
elif data_type == NewsDnnBaseDataReader.DictDataType["Wiki"]:
return self.get_data_wiki(cursor)
elif data_type == NewsDnnBaseDataReader.DictDataType["WikiAndTweet"]:
return self.get_data_wiki_and_tweet(cursor)
else:
LoggerHelper.critical('Unknown Data Type (data_type)')
return None
