def load_data(self):
LoanData.LCLoanData.load_data(self)
db_name = "loans_serialized_python%d.db" % sys.version_info.major
reload_db_files = utilities.reload_cached_file(db_name, self.args.stats.split(','))
if reload_db_files:
# We have to regenerate the loans using the base class
LoanData.LCLoanData.load_data(self)
self.info_msg("Creating memory sqlite3 from loans ...")
con, cursor = self.create_sqlite_memdb(self.loans)
self.info_msg("Creating memory sqlite3 from loans done ...")
self.info_msg("Creating cached sqlite3 %s ..." % db_name)
self.dump_sqlite_memdb(con, db_name)
self.info_msg("Creating cached sqlite3 %s ... done" % db_name)
else:
self.info_msg("Initializing sqlite3 from cached %s ..." % db_name)
con, cursor = self.load_sqlite_memdb(db_name)
self.info_msg("Initializing sqlite3 from cached %s done ..." % db_name)
self.con = con
self.cursor = cursor
def load_data(self):
loan_pickle_file = "loans_serialized_python%d.bin" % sys.version_info.major
loan_info_pickle_file = "loans_info_serialized_python%d.bin" % sys.version_info.major
reload_csv_files = utilities.reload_cached_file(loan_pickle_file, self.args.stats.split(','))
if not reload_csv_files:
if self.load_loans:
self.info_msg("Loading cached %s ..." % loan_pickle_file)
with open(loan_pickle_file, "rb") as fh:
self.loans = pickle.load(fh)
self.info_msg("Loading from cached %s done" % loan_pickle_file)
else:
self.info_msg("Not loading cached %s ..." % loan_pickle_file)
if self.load_loans_info:
self.info_msg("Loading cached %s ..." % loan_info_pickle_file)
with open(loan_info_pickle_file, "rb") as fh:
self.loans_info = pickle.load(fh)
self.info_msg("Loading from cached %s done" % loan_info_pickle_file)
else:
self.info_msg("Not loading cached %s ..." % loan_info_pickle_file)
else:
loans = []
loans_info = []
for csv_stats_file in self.args.stats.split(','):
if os.path.exists(csv_stats_file):
if zipfile.is_zipfile(csv_stats_file):
csv_file_name = os.path.splitext(os.path.basename(csv_stats_file))[0]
self.info_msg("Initializing from %s inside %s ..." % (csv_file_name, csv_stats_file))
with zipfile.ZipFile(csv_stats_file) as lc_stats_zip:
with lc_stats_zip.open(csv_file_name) as fh:
csv_reader = utilities.UnicodeCSVReader(fh, encoding="latin-1", wrapper=True)
self.parse_lc_csv(csv_reader, loans, loans_info)
else:
self.info_msg("Initializing from %s ..." % csv_stats_file)
with utilities.csv_open(csv_stats_file) as fh:
csv_reader = utilities.UnicodeCSVReader(fh, encoding="latin-1", wrapper=False)
self.parse_lc_csv(csv_reader, loans, loans_info)
else:
self.info_msg(csv_stats_file + " not found ... exiting")
sys.exit(-1)
with open(loan_pickle_file, "wb") as fh:
pickle.dump(loans, fh)
with open(loan_info_pickle_file, "wb") as fh:
pickle.dump(loans_info, fh)
self.loans = loans
self.loans_info = loans_info
