def load_data(self): LoanData.LCLoanData.load_data(self) db_name = "loans_serialized_python%d.db" % sys.version_info.major reload_db_files = utilities.reload_cached_file(db_name, self.args.stats.split(',')) if reload_db_files: # We have to regenerate the loans using the base class LoanData.LCLoanData.load_data(self) self.info_msg("Creating memory sqlite3 from loans ...") con, cursor = self.create_sqlite_memdb(self.loans) self.info_msg("Creating memory sqlite3 from loans done ...") self.info_msg("Creating cached sqlite3 %s ..." % db_name) self.dump_sqlite_memdb(con, db_name) self.info_msg("Creating cached sqlite3 %s ... done" % db_name) else: self.info_msg("Initializing sqlite3 from cached %s ..." % db_name) con, cursor = self.load_sqlite_memdb(db_name) self.info_msg("Initializing sqlite3 from cached %s done ..." % db_name) self.con = con self.cursor = cursor
def load_data(self): loan_pickle_file = "loans_serialized_python%d.bin" % sys.version_info.major loan_info_pickle_file = "loans_info_serialized_python%d.bin" % sys.version_info.major reload_csv_files = utilities.reload_cached_file(loan_pickle_file, self.args.stats.split(',')) if not reload_csv_files: if self.load_loans: self.info_msg("Loading cached %s ..." % loan_pickle_file) with open(loan_pickle_file, "rb") as fh: self.loans = pickle.load(fh) self.info_msg("Loading from cached %s done" % loan_pickle_file) else: self.info_msg("Not loading cached %s ..." % loan_pickle_file) if self.load_loans_info: self.info_msg("Loading cached %s ..." % loan_info_pickle_file) with open(loan_info_pickle_file, "rb") as fh: self.loans_info = pickle.load(fh) self.info_msg("Loading from cached %s done" % loan_info_pickle_file) else: self.info_msg("Not loading cached %s ..." % loan_info_pickle_file) else: loans = [] loans_info = [] for csv_stats_file in self.args.stats.split(','): if os.path.exists(csv_stats_file): if zipfile.is_zipfile(csv_stats_file): csv_file_name = os.path.splitext(os.path.basename(csv_stats_file))[0] self.info_msg("Initializing from %s inside %s ..." % (csv_file_name, csv_stats_file)) with zipfile.ZipFile(csv_stats_file) as lc_stats_zip: with lc_stats_zip.open(csv_file_name) as fh: csv_reader = utilities.UnicodeCSVReader(fh, encoding="latin-1", wrapper=True) self.parse_lc_csv(csv_reader, loans, loans_info) else: self.info_msg("Initializing from %s ..." % csv_stats_file) with utilities.csv_open(csv_stats_file) as fh: csv_reader = utilities.UnicodeCSVReader(fh, encoding="latin-1", wrapper=False) self.parse_lc_csv(csv_reader, loans, loans_info) else: self.info_msg(csv_stats_file + " not found ... exiting") sys.exit(-1) with open(loan_pickle_file, "wb") as fh: pickle.dump(loans, fh) with open(loan_info_pickle_file, "wb") as fh: pickle.dump(loans_info, fh) self.loans = loans self.loans_info = loans_info