def drop_tables():
"""Drops the tables so you can start again fresh"""
try:
PatientMeasurement.drop_table()
Feature.drop_table()
except Exception as e:
print "Failed to drop tables", e
raise
with open(TRAINING_DATA_FILE_NAME, "rb") as csvfile:
reader = csv.reader(csvfile, delimiter="|")
next(reader, None)
row_list = list(reader)
number_of_rows = len(row_list)
start_index = 0
print "Inserting %s rows" % number_of_rows
# xrange does not include the final number but its fine because we index
# from 0
start_time = datetime.datetime.now()
upper_limit = int(ceil(number_of_rows / BATCH_SIZE))
for _ in xrange(upper_limit):
batch_start_time = datetime.datetime.now()
PatientMeasurement.insert_many(
yield_x_rows(row_list, start_index, BATCH_SIZE)
).execute()
batch_end_time = datetime.datetime.now()
batch_time_taken = \
(batch_end_time - batch_start_time).total_seconds()
print "Inserted %s rows in %s seconds" % (BATCH_SIZE, batch_time_taken)
start_index += BATCH_SIZE
rows_remaining = number_of_rows - start_index
estimated_time_remaining = \
(rows_remaining / BATCH_SIZE) * batch_time_taken
print "%d rows remaining, estimated time %d" % (
rows_remaining, estimated_time_remaining
)
end_time = datetime.datetime.now()
time_taken = \
(end_time - start_time).total_seconds()