def _initialize(self):
"""
Sub PJT 1에서 만든 Dataframe을 이용하여 DB를 초기화합니다.
"""
print("[*] Loading data...")
dataframes = self._load_dataframes()
print("[*] Initializing stores...")
models.Store.objects.all().delete()
# models.User.objects.all().delete()
models.Menu.objects.all().delete()
models.Bhour.objects.all().delete()
models.Review.objects.all().delete()
stores = dataframes["stores"]
stores_bulk = [
models.Store(
store_id=store.id,
store_name=store.store_name,
branch=store.branch,
area=store.area,
tel=store.tel,
address=store.address,
latitude=store.latitude,
longitude=store.longitude,
category=store.category,
) for store in stores.itertuples()
]
models.Store.objects.bulk_create(stores_bulk)
# users = dataframes["users"]
# users_bulk = [
# models.User(
# user_id=user.id,
# gender=user.gender,
# age=user.age
# )
# for user in users.itertuples()
# ]
# models.User.objects.bulk_create(users_bulk)
menus = dataframes["menus"]
menus_bulk = [
models.Menu(store_id=menu.store,
menu=menu.menu_name,
price=menu.price) for menu in menus.itertuples()
]
models.Menu.objects.bulk_create(menus_bulk)
reviews = dataframes["reviews"]
reviews_bulk = [
models.Review(
review_id=review.id,
store_id=review.store,
# user_id=review.user,
total_score=review.score,
content=review.content,
reg_time=review.reg_time) for review in reviews.itertuples()
]
models.Review.objects.bulk_create(reviews_bulk)
print("[+] Done")
def _initialize(self):
"""
Sub PJT 1에서 만든 Dataframe을 이용하여 DB를 초기화합니다.
"""
print("[*] Loading data...")
dataframes = self._load_dataframes()
#
# print("[*] Initializing stores...")
# models.Store.objects.all().delete()
# stores = dataframes["stores"]
# stores_bulk = [
# models.Store(
# id=store.id,
# store_name=store.store_name,
# branch=store.branch,
# area=store.area,
# tel=store.tel,
# address=store.address,
# latitude=store.latitude,
# longitude=store.longitude,
# category=store.category,
# )
# for store in stores.itertuples()
# ]
# models.Store.objects.bulk_create(stores_bulk)
#
#
# # menus info
# print("[*] Initializing menus...")
# models.Menu.objects.all().delete()
#
# menus = dataframes["menus"]
# menus_bulk = [
# models.Menu(
# id=menu.id,
# store_id=menu.store,
# menu_name=menu.menu_name,
# price=menu.price
#
# )
# for menu in menus.itertuples()
# ]
# models.Menu.objects.bulk_create(menus_bulk)
# users info
# print("[*] Initializing users...")
# models.User.objects.all().delete()
# users = dataframes["users"]
# users_bulk = [
# models.User(
# id=user.id,
# gender=user.gender,
# age=user.age
# )
# for user in users.itertuples()
# ]
# models.User.objects.bulk_create(users_bulk)
# models.Profile.objects.all().delete()
# profile = dataframes["users"]
# profile_bulk = [
# models.Profile(
# id=user.id,
# gender=user.gender,
# age=user.age
# )
# for user in profile.itertuples()
# ]
# models.Profile.objects.bulk_create(profile_bulk)
#
# reviews info
print("[*] Initializing reviews...")
models.Review.objects.all().delete()
reviews = dataframes["reviews"]
reviews_bulk = [
models.Review(
# id=review.id,
store_id=review.store,
user_id=review.user,
total_score=review.score,
content=review.content,
reg_time=review.reg_time
)
for review in reviews.itertuples()
]
models.Review.objects.bulk_create(reviews_bulk)
print("[+] Done")
def _initialize(self):
"""
Sub PJT 1에서 만든 Dataframe을 이용하여 DB를 초기화합니다.
"""
print("[*] Loading data...")
df = self._load_dataframes()
print("[*] Initializing stores...")
models.Store.objects.all().delete()
stores = df["stores"]
stores_bulk = [
models.Store(id=store.id,
store_name=store.store_name,
area=store.area,
tel=store.tel,
address=store.address,
latitude=store.latitude,
longitude=store.longitude,
category=store.category,
classification=store.classification,
price_mean=store.price_mean)
for store in stores.itertuples()
]
models.Store.objects.bulk_create(stores_bulk)
print("[+] Stores Complete")
print("[*] Initializing users...")
models.User.objects.all().delete()
users = df['users']
users_bulk = [
models.User(id=user.id,
gender=user.gender,
age=user.age,
grade=user.grade,
vendor=user.vendor,
refresh_token=user.refresh_token)
for user in users.itertuples()
]
models.User.objects.bulk_create(users_bulk)
print("[+] Users Complete")
print("[*] Initializing reviews...")
models.Review.objects.all().delete()
reviews = df['reviews']
reviews_bulk = [
models.Review(id=review.id,
store=models.Store.objects.get(id=review.store),
user=models.User.objects.get(id=review.user),
score=review.score,
content=review.content,
reg_time=review.reg_time,
taste=review.taste,
service=review.service,
price_satisfaction=review.price_satisfaction,
interior=review.interior)
for review in reviews.itertuples()
]
models.Review.objects.bulk_create(reviews_bulk)
print("[+] Reviews Complete")
print("[*] Initializing Menu...")
models.Menu.objects.all().delete()
menu = df['menu']
menu_bulk = [
models.Menu(id=m.id,
store=models.Store.objects.get(id=m.store),
menu_name=m.menu_name,
price=m.price) for m in menu.itertuples()
]
models.Menu.objects.bulk_create(menu_bulk)
print("[+] Menu Complete")
print("[*] Initializing BHour...")
models.BHour.objects.all().delete()
b_hours = df['bhours']
bhour_bulk = [
models.BHour(id=b_hour.id,
store=models.Store.objects.get(id=b_hour.store),
type=b_hour.type,
week_type=b_hour.week_type,
mon=b_hour.mon,
tue=b_hour.tue,
wed=b_hour.wed,
thu=b_hour.thu,
fri=b_hour.fri,
sat=b_hour.sat,
sun=b_hour.sun,
start_time=b_hour.start_time,
end_time=b_hour.end_time,
etc=b_hour.etc) for b_hour in b_hours.itertuples()
]
models.BHour.objects.bulk_create(bhour_bulk)
print("[+] BHour Complete")
print("[+] Done")
def _initialize(self):
"""
Sub PJT 1에서 만든 Dataframe을 이용하여 DB를 초기화합니다.
"""
print("[*] Loading data...")
dataframes = self._load_dataframes()
print("[*] Initializing stores...")
models.Store.objects.all().delete()
stores = dataframes["stores"]
stores_bulk = [
models.Store(
id=store.id,
store_name=store.store_name,
branch=store.branch,
area=store.area,
tel=store.tel,
address=store.address,
latitude=store.latitude,
longitude=store.longitude,
category=store.category,
menu=store.menu,
bhour=store.bhour,
) for store in stores.itertuples()
]
models.Store.objects.bulk_create(stores_bulk)
print("[*] Initializing reviews...")
models.Review.objects.all().delete()
reviews = dataframes["reviews"]
reviews_bulk = [
models.Review(
storeid=review.store,
userid=review.user,
score=review.score,
content=review.content,
regtime=review.reg_time,
gender=review.gender,
bornyear=review.born_year,
) for review in reviews.itertuples()
]
models.Review.objects.bulk_create(reviews_bulk)
print("[*] Initializing tourspot...")
models.TourSpot.objects.all().delete()
print(dataframes)
tourspots = dataframes["tourspot"]
tourspots_bulk = [
models.TourSpot(
addr1=tourspot.addr1,
addr2=tourspot.addr2,
areacode=tourspot.areacode,
cat1=tourspot.cat1,
cat2=tourspot.cat2,
cat3=tourspot.cat3,
content_id=tourspot.content_id,
content_type_id=tourspot.content_type_id,
first_image=tourspot.first_image,
first_image2=tourspot.first_image2,
mapx=tourspot.mapx,
mapy=tourspot.mapy,
sigungucode=tourspot.sigungucode,
tel=tourspot.tel,
title=tourspot.title,
readcount=tourspot.readcount,
) for tourspot in tourspots.itertuples()
]
models.TourSpot.objects.bulk_create(tourspots_bulk)
print("[+] Done")
print("[*] Initializing Course...")
models.Course.objects.all().delete()
print(dataframes)
course = dataframes["course"]
course_bulk = [
models.Course(
addr1=course.addr1,
addr2=course.addr2,
areacode=course.areacode,
cat1=course.cat1,
cat2=course.cat2,
cat3=course.cat3,
content_id=course.content_id,
content_type_id=course.content_type_id,
first_image=course.first_image,
first_image2=course.first_image2,
mapx=course.mapx,
mapy=course.mapy,
sigungucode=course.sigungucode,
tel=course.tel,
title=course.title,
readcount=course.readcount,
) for course in course.itertuples()
]
models.Course.objects.bulk_create(course_bulk)
print("[+] Done")
print("[*] Initializing Recommand...")
models.Recommand.objects.all().delete()
print(dataframes)
recommandspots = dataframes["recommandspot"]
recommandspot_bulk = [
models.Recommand(
contentid=recommandspots.contentid,
contenttypeid=recommandspots.contenttypeid,
subcontentid=recommandspots.subcontentid,
subdetailalt=recommandspots.subdetailalt,
subdetailimg=recommandspots.subdetailimg,
subdetailoverview=recommandspots.subdetailoverview,
subname=recommandspots.subname,
subnum=recommandspots.subnum,
) for recommandspots in recommandspots.itertuples()
]
models.Recommand.objects.bulk_create(recommandspot_bulk)
print("[+] Done")
print("[*] Initializing Overview...")
models.Overview.objects.all().delete()
print(dataframes)
overviews = dataframes["overviews"]
overviews_bulk = [
models.Overview(
contentid=overviews.contentid,
contenttypeid=overviews.contenttypeid,
homepage=overviews.homepage,
overview=overviews.overview,
tel=overviews.tel,
title=overviews.title,
) for overviews in overviews.itertuples()
]
models.Overview.objects.bulk_create(overviews_bulk)
print("[+] Done")
def _initialize(self):
'''
기존의 dataframe pkl파일을 읽어와서 DB에 저장합니다.
'''
print("Loading part data")
cur_file = Path(settings.BASE_DIR) / "crawling" / "data" / "part.p"
with open(cur_file, 'rb') as f:
part_list = pickle.load(f)
print("complete")
print("Loading set data")
cur_file = Path(settings.BASE_DIR) / "crawling" / "data" / "set.p"
with open(cur_file, 'rb') as f:
set_list = pickle.load(f)
cur_file = Path(settings.BASE_DIR) / "crawling" / "data" / "set2.p"
with open(cur_file, 'rb') as f:
set_list2 = pickle.load(f)
print("complete")
print("Loading color data")
cur_file = Path(settings.BASE_DIR) / "crawling" / "data" / "color.p"
with open(cur_file, 'rb') as f:
color_list = pickle.load(f)["results"]
print("complete")
print("Loading part_categories data")
cur_file = Path(
settings.BASE_DIR) / "crawling" / "data" / "part_categories.p"
with open(cur_file, 'rb') as f:
part_categories_list = pickle.load(f)["results"]
print("complete")
print("Loading theme data")
cur_file = Path(settings.BASE_DIR) / "crawling" / "data" / "theme.p"
with open(cur_file, 'rb') as f:
theme_list = pickle.load(f)
print("complete")
print("Loading setpart data")
cur_file = Path(settings.BASE_DIR) / "crawling" / "data" / "setpart.p"
with open(cur_file, 'rb') as f:
setpart_list = pickle.load(f)
print("complete")
print("Loading user data")
cur_file = Path(settings.BASE_DIR) / "crawling" / "data" / "user.p"
with open(cur_file, 'rb') as f:
user_list = pickle.load(f)
print("complete")
print("Loading review data")
cur_file = Path(settings.BASE_DIR) / "crawling" / "data" / "review.p"
with open(cur_file, 'rb') as f:
review_list = pickle.load(f)
print("complete")
print("[*] Delete all data...")
# DB에 저장된 정보를 모두 지워 초기화해 줍니다.
models.CustomUser.objects.all().delete()
models.Theme.objects.all().delete()
models.LegoSet.objects.all().delete()
models.OfficialMapping.objects.all().delete()
models.Category.objects.all().delete()
models.Review.objects.all().delete()
models.LegoPart.objects.all().delete()
models.Color.objects.all().delete()
models.UserPart.objects.all().delete()
models.SetPart.objects.all().delete()
print("[+] Done")
print("[*] Initializing categories...")
categories_bulk = [
models.Category(id=category["id"],
name=category["name"],
part_count=category["part_count"])
for category in part_categories_list
]
models.Category.objects.bulk_create(categories_bulk)
print("[+] Done")
print("[*] Initializing colors...")
colors_bulk = [
models.Color(
id=color["id"],
name=color["name"],
rgb=color["rgb"],
bricklink_ids="|".join(
map(str, color["external_ids"]["BrickLink"]["ext_ids"]))
if color["external_ids"].get("BrickLink")
and color["external_ids"]["BrickLink"].get("ext_ids") else "",
bricklink_descrs="|".join(
color["external_ids"]["BrickLink"]["ext_descrs"][0])
if color["external_ids"].get("BrickLink") and
color["external_ids"]["BrickLink"].get("ext_descrs") else "",
official_ids="|".join(
map(str, color["external_ids"]["LEGO"]["ext_ids"]))
if color["external_ids"].get("LEGO")
and color["external_ids"]["LEGO"].get("ext_ids") else "",
official_descrs="|".join(
color["external_ids"]["LEGO"]["ext_descrs"][0])
if color["external_ids"].get("LEGO")
and color["external_ids"]["LEGO"].get("ext_descrs") else "",
) for color in color_list
]
models.Color.objects.bulk_create(colors_bulk)
models.Color.objects.create(id=9999, name="not a color", rgb="000000")
models.Color.objects.create(id=-1, name="Unknown", rgb="0033B2")
print("[+] Done")
print("[*] Initializing themes...")
themes_bulk = [
models.Theme(id=theme["id"],
parent_id=theme["parent_id"],
name=theme["name"],
root_id=theme["root_id"]) for theme in theme_list
]
models.Theme.objects.bulk_create(themes_bulk)
print("[+] Done")
print("[*] Initializing sets...")
# theme name n um_parts images review_count
sets_bulk = [
models.LegoSet(
theme_id=legoset[0],
name=legoset[1],
num_parts=legoset[2],
images=legoset[3],
review_count=legoset[4],
) for legoset in set_list
]
models.LegoSet.objects.bulk_create(sets_bulk)
print("[+] Done")
print("[*] Initializing official mapping table...")
mapping_table = [
models.OfficialMapping(id=v["set_num"], lego_set_id=i)
for i, v in enumerate(set_list2, 1)
]
models.OfficialMapping.objects.bulk_create(mapping_table)
print("[+] Done")
print("[*] Initializing lego parts...")
lego_part_bulk = [
models.LegoPart(
id=part["part_num"],
name=part["name"],
category_id=part["part_cat_id"],
image=part["part_img_url"] if part["part_img_url"] else "",
bricklink_ids="|".join(part["external_ids"]["BrickLink"])
if part["external_ids"].get("BrickLink") else "",
official_ids="|".join(part["external_ids"]["LEGO"])
if part["external_ids"].get("LEGO") else "")
for part in part_list
]
models.LegoPart.objects.bulk_create(lego_part_bulk)
print("[+] Done")
print("[*] Initializing setparts...")
# inventory_id part_num color_id quantity
setpart_bulk = [
models.SetPart(lego_set_id=part[0],
quantity=part[3],
color_id=part[2],
part_id=part[1]) for part in setpart_list
]
models.SetPart.objects.bulk_create(setpart_bulk)
print("[+] Done")
print("[*] Initializing users...")
# id username nickname age gender review_count
user_bulk = [
models.CustomUser(id=user[0],
username=user[1],
nickname=user[1],
age=user[3],
gender=user[4],
review_count=user[5]) for user in user_list
]
models.CustomUser.objects.bulk_create(user_bulk)
print("[+] Done")
print("[*] Initializing reviews...")
# score user_id lego_set_id
review_bulk = [
models.Review(score=review[0],
user_id=review[1],
lego_set_id=review[2]) for review in review_list
]
models.Review.objects.bulk_create(review_bulk)
print("[+] Done")
def _initialize(self):
"""
Sub PJT 1에서 만든 Dataframe을 이용하여 DB를 초기화합니다.
"""
print("[*] Loading data...")
dataframes = self._load_dataframes()
# print("[*] Initializing stores...")
# models.Store.objects.all().delete()
# stores = dataframes["stores"]
# stores_bulk = [
# models.Store(
# id=store.id,
# store_name=store.store_name,
# branch=store.branch,
# area=store.area,
# tel=store.tel,
# address=store.address,
# latitude=store.latitude,
# longitude=store.longitude,
# category=store.category,
# reviewCnt=store.reviewCnt,
# image=store.image
# )
# for store in stores.itertuples()
# ]
# models.Store.objects.bulk_create(stores_bulk)
# print("[+] Done")
# print("[*] Initializing users...")
# models.User.objects.all().delete()
# u.objects.all().delete()
# models.Account.objects.all().delete()
# users = dataframes["users"]
# users_bulk = []
# accounts_bulk = []
# auth_bulk = []
# for user in users.itertuples():
# users_bulk.append(models.User(
# id=user.id,
# gender=user.gender,
# age=user.age,
# ))
# auth_bulk.append(u(
# id=user.id,
# password='******',
# is_superuser=0,
# username='******' + str(user.id),
# first_name='',
# last_name='',
# email='',
# is_staff=0,
# is_active=0
# ))
# models.User.objects.bulk_create(users_bulk)
# u.objects.bulk_create(auth_bulk)
# print("[+] Done")
# print("[*] Initializing accounts...")
# accounts_bulk = [
# models.Account(
# id=index+1,
# gender=user.gender,
# age=user.age,
# user_id=user.id
# )
# for index, user in enumerate(users.itertuples())
# ]
# models.Account.objects.bulk_create(accounts_bulk)
# print("[+] Done")
print("[*] Initializing menus...")
models.Menu.objects.all().delete()
menus = dataframes["menus"]
menus_bulk = [
models.Menu(
id=menu.id,
store_id=menu.store,
menu_name=menu.menu_name,
price=menu.price
if math.isnan(float(menu.price)) == False else 0,
) for menu in menus.itertuples()
]
models.Menu.objects.bulk_create(menus_bulk)
print("[+] Done")
print("[*] Initializing reviews...")
models.Review.objects.all().delete()
reviews = dataframes["reviews"]
reviews_bulk = [
models.Review(
id=index + 1,
store_id=review.store,
user_id=review.user,
score=review.score,
content=review.content,
reg_time=review.reg_time,
) for index, review in enumerate(reviews.itertuples())
]
models.Review.objects.bulk_create(reviews_bulk)
print("[+] Done")
print("[*] Initializing bhours...")
models.Bhour.objects.all().delete()
bhours = dataframes["bhours"]
bhours_bulk = [
models.Bhour(
id=index + 1,
store_id=bhour.store,
week_type=bhour.week_type,
bhour_type=bhour.type,
mon=bhour.mon if math.isnan(float(bhour.mon)) == False else 0,
tue=bhour.tue if math.isnan(float(bhour.tue)) == False else 0,
wed=bhour.wed if math.isnan(float(bhour.wed)) == False else 0,
thu=bhour.thu if math.isnan(float(bhour.thu)) == False else 0,
fri=bhour.fri if math.isnan(float(bhour.fri)) == False else 0,
sat=bhour.sat if math.isnan(float(bhour.sat)) == False else 0,
sun=bhour.sun if math.isnan(float(bhour.sun)) == False else 0,
start_time=bhour.start_time,
end_time=bhour.end_time,
etc=bhour.etc)
for index, bhour in enumerate(bhours.itertuples())
]
models.Bhour.objects.bulk_create(bhours_bulk)
print("[+] Done")