def import_supplement_stack_and_compositions(engine):
print("Importing Supplement Stacks")
supplements_usersupplementstack = pd.read_sql_table(
"supplements_usersupplementstack", engine, index_col="id")
supplements_usersupplementstack = change_dataframe_nans_to_none(
supplements_usersupplementstack)
local_store[
"supplements_usersupplementstack"] = supplements_usersupplementstack
attributes_to_import = [
"modified",
"uuid",
]
for index, details in supplements_usersupplementstack.iterrows():
user = get_matching_user(details["user_id"])
name = details["name"]
defaults = details[attributes_to_import].to_dict()
SupplementStack.objects.update_or_create(user=user,
name=name,
defaults=defaults)
# okay, now that all the stacks have been imported ... let's add the actual compositions
supplements_usersupplementstackcomposition = pd.read_sql_table(
"supplements_usersupplementstackcomposition", engine, index_col="id")
supplements_usersupplementstackcomposition = change_dataframe_nans_to_none(
supplements_usersupplementstackcomposition)
local_store[
"supplements_usersupplementstackcomposition"] = supplements_usersupplementstackcomposition
attributes_to_import = [
"modified",
"uuid",
"quantity",
]
for index, details in supplements_usersupplementstackcomposition.iterrows(
):
user = get_matching_user(details["user_id"])
supplement = get_matching_supplement(details["supplement_id"])
supplement_stack = get_matching_supplement_stack(details["stack_id"])
# omfg, this is so clean. love pandas
defaults = details[attributes_to_import].to_dict()
SupplementStackComposition.objects.update_or_create(
user=user,
supplement=supplement,
stack=supplement_stack,
defaults=defaults)
def import_legacy_sleep_log(engine):
print("Importing Sleep")
df = pd.read_sql_table("events_sleeplog", engine, index_col="id")
df = change_dataframe_nans_to_none(df)
attributes_to_import = [
"uuid",
"source",
"start_time",
"end_time",
"modified",
]
for index, details in df.iterrows():
user = get_matching_user(details["user_id"])
defaults = details[attributes_to_import].to_dict()
defaults["source"] = "legacy_import"
start_time = defaults.pop("start_time")
end_time = defaults.pop("end_time")
sleep_log, _ = SleepLog.objects.update_or_create(user=user,
start_time=start_time,
end_time=end_time,
defaults=defaults)
def import_legacy_productivity(engine):
print("Importing Productivity")
productivity_df = pd.read_sql_table("events_dailyproductivitylog",
engine,
index_col="id")
productivity_df = change_dataframe_nans_to_none(productivity_df)
local_store["productivity_df"] = productivity_df
attributes_to_import = [
"uuid",
"source",
"date",
"very_productive_time_minutes",
"productive_time_minutes",
"neutral_time_minutes",
"distracting_time_minutes",
"very_distracting_time_minutes",
]
for index, details in productivity_df.iterrows():
user = get_matching_user(details["user_id"])
defaults = details[attributes_to_import].to_dict()
defaults["source"] = "legacy_import"
date = defaults.pop("date")
productivity_log, _ = DailyProductivityLog.objects.update_or_create(
user=user, date=date, defaults=defaults)
def import_legacy_activities_logs(engine):
print("Importing Activities Logs")
df = pd.read_sql_table("events_useractivity", engine, index_col="id")
df = change_dataframe_nans_to_none(df)
local_store["activities_df"] = df
print("Finished Importing Table from Heroku")
activities_log_df = pd.read_sql_table("events_useractivitylog",
engine,
index_col="id")
activities_log_df = change_dataframe_nans_to_none(activities_log_df)
local_store["activities_log_df"] = df
attributes_to_import = [
"modified",
"uuid",
"source",
"duration_minutes",
]
for index, details in activities_log_df.iterrows():
time = details["time"]
user_id = details["user_id"]
user = get_matching_user(user_id)
activity_id = details["user_activity_id"]
activity = get_matching_activity(activity_id)
defaults = details[attributes_to_import].to_dict()
defaults["source"] = "legacy_import"
ActivityLog.objects.update_or_create(time=time,
user=user,
activity=activity,
defaults=defaults)
def import_legacy_activities(engine):
print("Importing Activities")
df = pd.read_sql_table("events_useractivity", engine, index_col="id")
df = change_dataframe_nans_to_none(df)
local_store["activities_df"] = df
print("Finished Importing Table from Heroku")
activities_log_df = pd.read_sql_table("events_useractivitylog",
engine,
index_col="id")
activities_log_df = change_dataframe_nans_to_none(activities_log_df)
local_store["activities_log_df"] = df
activities_used = activities_log_df["user_activity_id"].unique()
# most activities you auto-created by default don't need to be used. don't import those
activities_to_import = df.loc[activities_used]
attributes_to_import = [
"modified",
"uuid",
"name",
"is_negative_activity",
"is_all_day_activity",
]
for index, details in activities_to_import.iterrows():
user = get_matching_user(details["user_id"])
defaults = details[attributes_to_import].to_dict()
name = defaults.pop("name")
instance, _ = Activity.objects.update_or_create(user=user,
name=name,
defaults=defaults)
def import_legacy_supplements_log(engine):
print("Importing Supplements Log")
df = pd.read_sql_table("events_supplementlog", engine, index_col="id")
df = change_dataframe_nans_to_none(df)
local_store["supplement_log_df"] = df
print("Finished Importing Table from Heroku")
last_dt = df["time"].max()
# convoluted crap to deal with import was too slow, now handle via batch updates instead
distinct_users = df["user_id"].unique()
for user_id in distinct_users:
user = get_matching_user(user_id)
# wipe out the previous imports to do it one much larger operation, otherwise it
# was taking too long on production
SupplementLog.objects.filter(user=user, time__lte=last_dt).delete()
attributes_to_import = [
"modified",
"uuid",
"quantity",
"duration_minutes",
"notes",
]
supplement_logs_to_create = []
for index, details in df.iterrows():
user = get_matching_user(details["user_id"])
supplement = get_matching_supplement(details["supplement_id"])
time = details["time"]
defaults = details[attributes_to_import].to_dict()
defaults["source"] = "legacy_import"
instance = SupplementLog(user=user,
supplement=supplement,
time=time,
**defaults)
supplement_logs_to_create.append(instance)
if index % 500 == 0:
print(f"Adding {instance} to Supplement Logs to Create")
print(f"Bulk Creating Supplements")
SupplementLog.objects.bulk_create(supplement_logs_to_create, 500)
print(f"Finished Creating Supplements")
def import_legacy_supplements(engine):
print("Importing Supplements")
df = pd.read_sql_table("supplements_supplement", engine, index_col="id")
df = change_dataframe_nans_to_none(df)
local_store["supplements_df"] = df
attributes_to_import = [
"uuid",
"name",
"modified",
]
for index, details in df.iterrows():
user = get_matching_user(details["user_id"])
defaults = details[attributes_to_import].to_dict()
name = defaults.pop("name")
instance, _ = Supplement.objects.update_or_create(user=user,
name=name,
defaults=defaults)
def import_legacy_mood_logs(engine):
print("Importing Mood Logs")
df = pd.read_sql_table("events_usermoodlog", engine, index_col="id")
df = change_dataframe_nans_to_none(df)
local_store["mood_log"] = df
attributes_to_import = [
"modified",
"uuid",
"source",
"notes",
]
for index, details in df.iterrows():
user = get_matching_user(details["user_id"])
mental_value = details["value"]
time = details["time"]
defaults = details[attributes_to_import].to_dict()
defaults["source"] = "legacy_import"
instance, _ = WellBeingLog.objects.update_or_create(
user=user, time=time, mental_value=mental_value, defaults=defaults)