def test_nested_obj(self):
class Course(object):
def __init__(self, code, name):
self.code = code
self.name = name
class Student(object):
def __init__(self, name, age, course):
self.name = name
self.age = age
self.course = course
course = Course("CS201", "Data Structures")
student = Student("Danish", 24, course)
query = ["name", "age", {
"course": [
"code",
"name",
]
}]
self.assertEqual(
dictfier.dictfy(student, query), {
"name": "Danish",
"age": 24,
"course": {
"code": "CS201",
"name": "Data Structures"
}
})
def get_model_rec(self, model, rec_id, **params):
records = request.env[model].search([])
if "query" in params:
query = json.loads(params["query"])
else:
query = records.fields_get_keys()
if "exclude" in params:
exclude = json.loads(params["exclude"])
for field in exclude:
if field in query:
field_to_exclude = query.index(field)
query.pop(field_to_exclude)
record = records.browse(rec_id).ensure_one()
data = dictfier.dictfy(
record,
query,
flat_obj=flat_obj,
nested_flat_obj=nested_flat_obj,
nested_iter_obj=nested_iter_obj
)
return http.Response(
json.dumps(data),
status=200,
mimetype='application/json'
)
def test_global_dictfy_config_with_parent_and_field_name_params(self):
# Customize how dictfier obtains flat obj,
# nested flat obj and nested iterable obj
# per dictfy call (global)
class Book(object):
def __init__(self, title, publish_date):
self.title = title
self.publish_date = publish_date
class Course(object):
def __init__(self, code, name, books):
self.code = code
self.name = name
self.books = books
class Student(object):
def __init__(self, name, age, course):
self.name = name
self.age = age
self.course = course
book1 = Book("Advanced Data Structures", "2018")
book2 = Book("Basic Data Structures", "2010")
course = Course("CS201", "Data Structures", [book1, book2])
student = Student("Danish", 24, course)
query = [
"name", "age", {
"course":
["name", "code", {
"books": [["title", "publish_date"]]
}]
}
]
self.assertEqual(
dictfier.dictfy(student,
query,
flat_obj=lambda obj, parent, field_name: getattr(
parent, field_name),
nested_iter_obj=lambda obj, parent, field_name:
getattr(parent, field_name),
nested_flat_obj=lambda obj, parent, field_name:
getattr(parent, field_name)), {
'name': 'Danish',
'age': 24,
'course': {
'name':
'Data Structures',
'code':
'CS201',
'books': [{
'title': 'Advanced Data Structures',
'publish_date': '2018'
}, {
'title': 'Basic Data Structures',
'publish_date': '2010'
}]
}
})
def test_empty_query_against_flat_nested_and_iterable_obj(self):
class Book(object):
def __init__(self, title, publish_date):
self.title = title
self.publish_date = publish_date
class Course(object):
def __init__(self, code, name, book):
self.code = code
self.name = name
self.book = book
class Student(object):
def __init__(self, name, age, courses):
self.name = name
self.age = age
self.courses = courses
book1 = Book("Advanced Data Structures", "2018")
book2 = Book("Basic Data Structures", "2010")
course1 = Course("CS201", "Data Structures", book1)
course2 = Course("CS205", "Computer Networks", book2)
courses = [course1, course2]
student = Student("Danish", 24, courses)
query1 = [] # Empty outer flat query
query2 = [[]] # Empty outer iterable query
# Empty nested flat query
query3 = [{"book": []}]
# Empty nested iterable query
query4 = [{"courses": [[]]}]
self.assertEqual(dictfier.dictfy(student, query1), {})
self.assertEqual(dictfier.dictfy(courses, query2), [{}, {}])
self.assertEqual(dictfier.dictfy(course1, query3), {"book": {}})
self.assertEqual(dictfier.dictfy(student, query4),
{"courses": [{}, {}]})
def write_csv(file_name, rows, fieldnames):
"""
This is a generic method for writing out the .csv files.
"""
# Open the .csv file for writing
with open(file_name, mode='w') as out_file:
# Write out the headings line
writer = csv.DictWriter(out_file, fieldnames=fieldnames)
writer.writeheader()
# Iterate through the rows, writing out the data
for cur in rows:
writer.writerow(dictfier.dictfy(cur, fieldnames))
def test_newfield_api(self):
class Student(object):
def __init__(self, name, age):
self.name = name
self.age = age
student = Student("Danish", 24)
query = ["name", "age", {"school": dictfier.newfield("St Patrick")}]
self.assertEqual(dictfier.dictfy(student, query), {
'name': 'Danish',
'age': 24,
'school': 'St Patrick'
})
def test_flat_obj(self):
class Student(object):
def __init__(self, name, age):
self.name = name
self.age = age
student = Student("Danish", 24)
query = [
"name",
"age",
]
self.assertEqual(dictfier.dictfy(student, query), {
"name": "Danish",
"age": 24
})
def test_objfield_api(self):
class Student(object):
def __init__(self, name, age):
self.name = name
self.age = age
student = Student("Danish", 24)
query = [
"name",
{
"age_in_years": dictfier.objfield("age")
},
]
self.assertEqual(dictfier.dictfy(student, query), {
'name': 'Danish',
'age_in_years': 24
})
def to_representation(self, instance):
try:
request = self.context['request']
except KeyError:
warnings.warn('Context does not have access to request.')
return super().to_representation(instance)
if self.query_param_name in request.query_params:
query = json.loads(request.query_params[self.query_param_name])
data = dictfier.dictfy(
instance,
query,
flat_obj=self.flat_field,
nested_flat_obj=self.nested_flat_field,
nested_iter_obj=self.nested_iter_field,
)
return data
return super().to_representation(instance)
def test_custom_iterable_nested_obj(self):
# Customize how dictfier obtains nested iterable obj
# per nested field
class Course(object):
def __init__(self, code, name):
self.code = code
self.name = name
class Student(object):
def __init__(self, name, age, courses):
self.name = name
self.age = age
self.courses = courses
course1 = Course("CS201", "Data Structures")
course2 = Course("CS205", "Computer Networks")
student = Student("Danish", 24, [course1, course2])
query = [
"name", "age", {
"courses":
dictfier.useobj(lambda obj: obj.courses, [[
"code",
"name",
]])
}
]
self.assertEqual(
dictfier.dictfy(student, query), {
'name':
'Danish',
'age':
24,
'courses': [{
'code': 'CS201',
'name': 'Data Structures'
}, {
'code': 'CS205',
'name': 'Computer Networks'
}]
})
def test_objfield_api_with_call_kwarg(self):
class Student(object):
def __init__(self, name, age):
self.name = name
self.age = age
def age_in_months(self):
return self.age * 12
student = Student("Danish", 24)
query = [
"name",
{
"months": dictfier.objfield("age_in_months", call=True)
},
]
self.assertEqual(dictfier.dictfy(student, query), {
'name': 'Danish',
'months': 288
})
def test_useobj_api(self):
class Student(object):
def __init__(self, name, age):
self.name = name
self.age = age
student = Student("Danish", 24)
def age_in_months(obj):
# Do the computation here then return the result
return obj.age * 12
query = [
"name",
{
"age_in_months": dictfier.useobj(age_in_months)
},
]
self.assertEqual(dictfier.dictfy(student, query), {
'name': 'Danish',
'age_in_months': 288
})
def test_iterable_nested_obj(self):
class Course(object):
def __init__(self, code, name):
self.code = code
self.name = name
class Student(object):
def __init__(self, name, age, courses):
self.name = name
self.age = age
self.courses = courses
course1 = Course("CS201", "Data Structures")
course2 = Course("CS205", "Computer Networks")
student = Student("Danish", 24, [course1, course2])
query = ["name", "age", {
"courses": [[
"code",
"name",
]]
}]
self.assertEqual(
dictfier.dictfy(student, query), {
'name':
'Danish',
'age':
24,
'courses': [{
'code': 'CS201',
'name': 'Data Structures'
}, {
'code': 'CS205',
'name': 'Computer Networks'
}]
})
def test_custom_nested_obj(self):
# Customize how dictfier obtains nested flat obj
# per nested field
class Course(object):
def __init__(self, code, name):
self.code = code
self.name = name
class Student(object):
def __init__(self, name, age, course):
self.name = name
self.age = age
self.course = course
course = Course("CS201", "Data Structures")
student = Student("Danish", 24, course)
query = [
"name",
"age",
{
"course":
dictfier.useobj(
lambda obj: obj.course,
["name", "code"] # This is a query
)
}
]
self.assertEqual(
dictfier.dictfy(student, query), {
'name': 'Danish',
'age': 24,
'course': {
'name': 'Data Structures',
'code': 'CS201'
}
})
def post_model_data(self, model, **post):
try:
data = post['data']
except KeyError:
_logger.exception(
"'data' parameter is not found on POST request"
)
if "context" in post:
context = post["context"]
record = request.env[model].with_context(**context)\
.create(data)
else:
record = request.env[model].create(data)
if model == 'sgu.order':
data = dictfier.dictfy(
record,
['name', 'id', 'total', 'state', 'process'],
flat_obj=flat_obj,
nested_flat_obj=nested_flat_obj,
nested_iter_obj=nested_iter_obj
)
return data
return record.id
def GetProductDetail(product_url):
driver = webdriver.Chrome(
executable_path=
"C:\\Users\\tlhqu\\Downloads\\chromedriver_win32\\chromedriver.exe",
chrome_options=chrome_options)
print(" >> ", product_url)
driver.get(product_url)
try:
WebDriverWait(driver, 1).until(
EC.presence_of_element_located((By.CLASS_NAME, 'container')))
print("Page is ready!")
except TimeoutException:
print("Loading took too much")
scrolls = 4
while True:
scrolls -= 1
time.sleep(1)
driver.execute_script("window.scrollBy(0, 1080)")
if scrolls < 0:
break
_page = BeautifulSoup(driver.page_source, "html.parser")
product_list = []
title = _page.findAll("div", {"class": "_3ZV7fL"})[0].text
rating = _page.findAll("div", {"class": "_3WXigY _22cC7R"})
rating = rating[0].text if len(rating) > 0 else ""
price_current = _page.findAll("div", {"class": "AJyN7v"})
price_current = price_current[0].text if len(price_current) > 0 else ""
price_origin = _page.findAll("div", {"class": "bBOoii"})
price_origin = price_origin[0].text if len(price_origin) > 0 else ""
price_discount = _page.findAll("div", {"class": "_3ghar9"})
price_discount = price_discount[0].text if len(price_discount) > 0 else ""
product = Product(title, rating, price_current, price_origin,
price_discount)
product_list.append(product)
_porperty = _page.findAll("div", {"class": "flex _1nb0l8 _2J8-Qs"})
porperty_list = []
for i in _porperty:
name = i.findAll(
"div", {"style": "margin-bottom: 8px; align-items: baseline;"})
for k in name:
porperty_name = k.label.text
list_value = []
value = k.findAll("button")
for j in value:
list_value.append(j.text)
porperty = Porperty(porperty_name, list_value)
porperty_list.append(porperty)
# store
store_list = []
store_link = "https://shopee.vn" + _page.findAll(
"div", {"class": "_3XjqaB"})[0].a["href"]
store_ava = _page.findAll("div",
{"class": "shopee-avatar _3sZdoP"})[0].img['src']
store_name = _page.findAll("div", {"class": "_3KP9-e"})[0].text
store_time = _page.findAll("div", {"class": "_2QJIt0"})[0].text
_inf = _page.findAll("div", {"class": "dos2U- eTrFKz"})
store_info = []
for inf in _inf:
inf_name = inf.label.text
inf_value = inf.span.text
list_info = {}
list_info[inf_name] = inf_value
store_info.append(list_info)
store = Store(store_link, store_ava, store_name, store_time, store_info)
store_list.append(store)
# thông tin sản phẩm
info_list = []
product_inf = _page.findAll("div", {"class": "UjOlxf"})
info = product_inf[0].findAll("div", {"class": "_2gVYdB"})
for i in info:
name_info = i.label.text
value_info = i.a
value_info = i.a.text if value_info is not None else i.div.text
productinfo = ProductInfo(name_info, value_info)
info_list.append(productinfo)
# hình ảnh
img_list = []
product_img = _page.findAll("div", {"class": "_2wBoeW V1Fpl5"})
for img in product_img:
img_link = (img["style"].split('"')[1])
link = Img(img_link)
img_list.append(link)
data = ShoppeeProductDetail(porperty_list, product_list, store_list,
info_list, img_list)
query = [{
"Porperties": [["Name", "Value"]]
}, {
"Products":
[["Title", "Rating", "PriceCurrent", "PriceOrigin", "PriceDiscount"]]
}, {
"Stores": [["Link", "Ava", "Name", "Time", "Info"]]
}, {
"ProductInfos": [["InfoName", "InfoValue"]]
}, {
"ImgLinks": [["ImgLink"]]
}]
std_info = dictfier.dictfy(data, query)
file_name = uuid.uuid4().hex + ".json"
completeName = os.path.join(os.getcwd(), file_name)
file = open(completeName, "w", encoding="utf8")
file.write(json.dumps(std_info))
file.close()
def GetProductDetail(product_url):
driver = webdriver.Chrome(executable_path = "C:\\Users\\tlhqu\\Downloads\\chromedriver_win32\\chromedriver.exe", chrome_options=chrome_options)
print(" >> ",product_url)
driver.get(product_url)
try:
WebDriverWait(driver, 1).until(EC.presence_of_element_located((By.ID , 'root')))
print("Page is ready!")
except TimeoutException:
print ("Loading took too much")
scrolls = 3
while True:
scrolls -= 1
time.sleep(1)
driver.execute_script("window.scrollBy(0, 1080)")
if scrolls < 0:
break
script = "document.querySelector(\"#product-detail > div.product-detail-tab > div > div.detail-tab-bar > ul > li:nth-child(3)\").click()"
driver.execute_script(script)
_page = BeautifulSoup(driver.page_source, "html.parser")
driver.quit()
i=_page.findAll("div",{"class":"glodetail-wrap"})[0]
list_pro = []
# tìm thông tin
title = i.findAll("div",{"class":"product-title"})[0].h1.text.strip()
rate = i.findAll("span",{"class":"overview-rating-average"})
rate = rate[0].text if len(rate) > 0 else ""
price_current = i.findAll("div",{"class":"product-price-current"})[0].span.text.strip()
price_original = i.findAll("div",{"class":"product-price-original"})
price_original = price_original[0].findAll("span",{"class":"product-price-value"}) if len(price_original)> 0 else ''
price_original =price_original[0].text.strip() if len(price_original) > 0 else ""
price_discount = i.findAll("div",{"class":"product-price-original"})
price_discount = price_discount[0].findAll("span",{"class":"product-price-mark"}) if len(price_discount) > 0 else ""
price_discount = price_discount[0].text.strip() if len(price_discount) >0 else ""
coupon = i.findAll("div",{"class":"coupon-mark-new"})
coupon = coupon[0].text.strip() if len(coupon) > 0 else ""
shipping = i.findAll("div",{"class":"product-shipping-price"})
shipping = shipping[0].text.strip() if len(shipping) >0 else ("Please select the country you want to ship from")
productDetail=ProductDetail(title,rate,price_current,price_original,price_discount,coupon,shipping)
list_pro.append(productDetail)
# thông tin store
list_store = []
store = i.findAll("div",{"class":"shop-name"})
store_name = store[0].text.strip() if len(store) > 0 else ""
store_link = store[0].a["href"] if len(store) > 0 else ""
store_positive = i.findAll("div",{"class":"positive-fdbk"})
store_positive = store_positive[0].text if len(store_positive) > 0 else ""
store_follower_num = i.findAll("div",{"class":"follower-num"})
store_follower_num = store_follower_num[0].text.strip() if len(store_follower_num) > 0 else ""
store_contact = i.findAll("div",{"class":"store-info-contact"})
store_contact = store_contact[0].a["href"] if len(store_contact) > 0 else ""
store_top = i.findAll("div",{"class":"top-seller-label"})
store_top = store_top[0].text.strip() if len(store_top) > 0 else ""
store = Store(store_name,store_link,store_positive,store_follower_num,store_contact,store_top)
list_store.append(store)
# thông tin sku: hình dạng, kích thước, size,...
sku_list =[]
sku_ = i.findAll("div",{"class":"sku-wrap"})[0].findAll('div',{"class":"sku-property"})
for j in sku_:
sku_name = j.findAll("div",{"class":"sku-title"})[0].text.strip()
sku_value = []
_sku_value = j.findAll("li",{"class":"sku-property-item"})
for k in _sku_value:
value = k.find('img')
if value is not None:
value_name = value['alt']
value_url = value['src'].replace(".jpg_50x50","")
else:
value_name = k.text.strip()
value_url = ''
list_value = {}
list_value["Name"] = value_name
list_value["Url"] = value_url
sku_value.append(list_value)
sku = SKU(sku_name,sku_value)
sku_list.append(sku)
# thông số kĩ thuật
list_detail = []
detail = i.findAll("ul",{"class":"product-specs-list util-clearfix"})[0].findAll("li")
for q in detail:
q=q.text.split(":")
detail_name = q[0]
detail_value = q[1]
detail = Specification(detail_name,detail_value)
list_detail.append(detail)
# # hình ảnh
list_img = []
image = i.findAll("div",{"class":"images-view-item"})
for index in image:
img_link = index.img['src'].replace(".jpg_50x50","")
img_name = index.img['alt']
# print(detail)
img = Img(img_name,img_link)
list_img.append(img)
data = DataProductDetail(list_pro,list_store,sku_list, list_img,list_detail)
query = [
{
"ProductDetails": [
[
"ProName",
"Rate",
"PriceCurrent",
"PriceOrigin",
"PriceDiscount",
"Coupon",
"Shipping"
]
]
},
{
"Stories": [
[
"StoreName",
"URL",
"StorePositive",
"StoreFollowerNum",
"StoreContact",
"StoreTop"
]
]
},
{
"SKUs": [
[
"SKUName",
"SKUValue"
]
]
},
{
"Specifications": [
[
"DetailName",
"DetailValue"
]
]
},
{
"Imgs": [
[
"ImgName",
"ImgLink"
]
]
}
]
std_info = dictfier.dictfy(data, query)
file_name = uuid.uuid4().hex + ".json"
completeName = os.path.join(os.getcwd(), file_name)
file = open(completeName, "w", encoding="utf8")
file.write(json.dumps(std_info))
file.close()
print(completeName)
def test_query_format_violation(self):
class Course(object):
def __init__(self, code, name):
self.code = code
self.name = name
class Student(object):
def __init__(self, name, age, courses):
self.name = name
self.age = age
self.courses = courses
course1 = Course("CS201", "Data Structures")
course2 = Course("CS205", "Computer Networks")
student = Student("Danish", 24, [course1, course2])
query1 = [
"name",
"age",
566 # FormatError
]
query2 = [
"name",
"age",
["name"] # FormatError
]
query3 = [
"name",
"age",
{
"courses": [
["code", "name"],
"name", # FormatError
]
}
]
query4 = [
"name",
"age",
{
"courses": [
"name", # FormatError
["code", "name"],
]
}
]
query5 = [
"name",
"age",
{
"class": "HBO" # TypeError
}
]
with self.assertRaises(dictfier.exceptions.FormatError):
dictfier.dictfy(student, query1)
with self.assertRaises(dictfier.exceptions.FormatError):
dictfier.dictfy(student, query2)
with self.assertRaises(dictfier.exceptions.FormatError):
dictfier.dictfy(student, query3)
with self.assertRaises(dictfier.exceptions.FormatError):
dictfier.dictfy(student, query4)
with self.assertRaises(TypeError):
dictfier.dictfy(student, query5)
def GetProductList(cate_url, max_num_pages):
list_product = []
url = cate_url
for ele in range(0, max_num_pages + 1):
driver = webdriver.Chrome(
executable_path=
"C:\\Users\\tlhqu\\Downloads\\chromedriver_win32\\chromedriver.exe",
chrome_options=chrome_options)
print(" >> ", url + str(ele))
driver.get(url + str(ele))
try:
WebDriverWait(driver, 1).until(
EC.presence_of_element_located((By.CLASS_NAME, '_2eoI9r')))
print("Page is ready!")
except TimeoutException:
print("Loading took too much")
continue
scrolls = 6
while True:
scrolls -= 1
time.sleep(1)
driver.execute_script("window.scrollBy(0, 1080)")
if scrolls < 0:
break
# Lấy html từ page source
_page = BeautifulSoup(driver.page_source, "html.parser")
driver.quit()
subcate_list = []
sub = _page.findAll("a",
{"class": "shopee-category-list__sub-category"})
for i in sub:
subcate_name = i.text
subcate_link = "https://shopee.vn" + i['href']
subcate_id = subcate_link.split(".")[-1]
subcate = SubCategory(subcate_name, subcate_link, subcate_id)
subcate_list.append(subcate)
product_list = []
product = _page.findAll("div", {"data-sqe": "item"})
for j in product:
product_url = "https://shopee.vn" + j.a['href']
product_name = j.findAll(
"div", {"class": "_1NoI8_ A6gE1J _1co5xN"})[0].text.strip()
# giá hiển thị
product_price_current = j.findAll(
"div", {"class": "_1w9jLI _1DGuEV _7uLl65"})[0].text
# giá hiển thị bị gạch
product_price_origin = j.findAll(
"div", {"class": "_1w9jLI _1y3E4O _2vQ-UF"})
product_price_origin = product_price_origin[0].text if len(
product_price_origin) > 0 else ""
# giá discount
product_price_discount = j.findAll("span", {"class": "percent"})
product_price_discount = product_price_discount[0].text if len(
product_price_discount) > 0 else ""
# thông tin như : mua giá bán buôn/ sỉ, rẻ vô địch ,...
product_price_text_1 = j.findAll("div", {"class", "Lj2P-3"})
product_price_text_1 = product_price_text_1[0].text if len(
product_price_text_1) > 0 else ""
product_price_text_2 = j.findAll("div", {"class", "_2CgHpG"})
product_price_text_2 = product_price_text_2[0].text if len(
product_price_text_2) > 0 else ""
product_price_text_3 = j.findAll("div", {"class", "E9F8gz"})
product_price_text_3 = product_price_text_3[0].text if len(
product_price_text_3) > 0 else ""
product_price_text_4 = j.findAll("div", {"class", "_1FKkT"})
product_price_text_4 = product_price_text_4[0].text if len(
product_price_text_4) > 0 else ""
product_price_text_5 = j.findAll("div", {"class", "_1tJDR9"})
product_price_text_5 = product_price_text_5[0].text if len(
product_price_text_5) > 0 else ""
# thông tn tỉnh/thành phố
product_location = j.findAll("div", {"class": "_41f1_p"})
product_location = product_location[0].text if len(
product_location) > 0 else ""
# Yêu thích
product_like = j.findAll("span", {"class": "_1DeDTg"})
product_like = product_like[0].text if len(
product_like) > 0 else ""
# thông tin có áp dung freehship
product_icon_freeship = j.findAll("div", {"class": "_3c2vFv"})
product_icon_freeship = "Yes" if len(
product_icon_freeship) > 0 else "No"
product_img_link = j.findAll(
"div", {"class": "_39-Tsj _1tDEiO"})[0].img['src']
product = Product(product_name, product_url, product_price_current,
product_price_origin, product_price_discount,
product_price_text_1, product_price_text_2,
product_price_text_3, product_price_text_4,
product_price_text_5, product_location,
product_icon_freeship, product_img_link,
product_like)
product_list.append(product)
data = DataProductList(product_list, subcate_list)
query = [{
"Products": [[
"Name", "URL", "PriceCurrent", "PriceOriginal",
"PriceDiscount", "PriceText1", "PriceText2", "PriceText3",
"PriceText4", "PriceText5", "Location", "Shipping", "ImgLink",
"ProductLike"
]]
}, {
"SubCategories": [["SubCateName", "SubCateURL", "SubCateID"]]
}]
std_info = dictfier.dictfy(data, query)
file_name = uuid.uuid4().hex + ".json"
completeName = os.path.join(os.getcwd(), file_name)
file = open(completeName, "w", encoding="utf8")
file.write(json.dumps(std_info))
file.close()
def GetProductList(cate_url, max_num_pages):
list_product = []
url = "https:" + cate_url
for ele in range(1, max_num_pages + 1):
driver = webdriver.Chrome(
executable_path=
"C:\\Users\\tlhqu\\Downloads\\chromedriver_win32\\chromedriver.exe",
chrome_options=chrome_options)
print(" >> ", url + str(ele))
driver.get(url + str(ele))
try:
WebDriverWait(driver, 1).until(
EC.presence_of_element_located(
(By.CLASS_NAME, 'product-container')))
print("Page is ready!")
except TimeoutException:
print("Loading took too much")
continue
scrolls = 6
while True:
scrolls -= 1
time.sleep(1)
driver.execute_script("window.scrollBy(0, 1080)")
if scrolls < 0:
break
# Lấy html từ page source
_page = BeautifulSoup(driver.page_source, "html.parser")
driver.quit()
scate_list = []
scate_name_field = "Subcate_name"
scate_link_field = "Subcate_link"
sub = _page.findAll("ul", {"class": "child-menu"})[0].findAll("li")
for i in sub:
scate_id = i['ae_object_value']
print(scate_id)
scate_name = i.a.text.strip()
scate_url = i.a["href"]
scate_str = SubCategory(scate_name, scate_url, scate_id)
scate_list.append(scate_str)
post = _page.findAll("li", {"class": "list-item"})
list_product_str = []
for i in post:
product_url = i.a["href"]
product_title = i.text.strip()
product_price_current = i.findAll("span",
{"class": "price-current"})
product_price_current = product_price_current[0].text.strip(
) if len(product_price_current) > 0 else ""
product_price_origin = i.findAll("span",
{"class": "price-original"})
product_price_origin = product_price_origin[0].text.strip(
) if len(product_price_origin) > 0 else ""
product_price_discount = i.findAll("span",
{"class": "price-discount"})
product_price_discount = product_price_discount[0].text.strip(
) if len(product_price_discount) > 0 else ""
product_shipping = i.findAll("span", {"class": "shipping-value"})
product_shipping = product_shipping[0].text.strip(
) if len(product_shipping) > 0 else ""
product_rate = i.findAll("span", {"class": "rating-value"})
product_rate = product_rate[0].text.strip(
) if len(product_rate) > 0 else ""
product_store = i.findAll("div", {"class": "item-store-wrap"})
product_store = product_store[0].a.text.strip(
) if len(product_store) > 0 else ""
product_store_link = i.findAll("div", {"class": "item-store-wrap"})
product_store_link = product_store_link[0].a['href'] if len(
product_store_link) > 0 else ""
product_img_link = i.findAll(
"div", {"class": "product-img"})[0].a.img["src"].replace(
".jpg_220x220xz", "")
product_img_name = i.findAll(
"div", {"class": "product-img"})[0].a.img["alt"]
product_str = Product(product_title, product_url,
product_price_current, product_price_origin,
product_price_discount, product_shipping,
product_rate, product_store,
product_store_link, product_img_link)
list_product_str.append(product_str)
data = DataProductList(list_product_str, scate_list)
query = [{
"Products": [[
"Name", "URL", "PriceCurrent", "PriceOriginal",
"PriceDiscount", "Shipping", "Rate", "StoreName", "StoreLink",
"ImgLink"
]]
}, {
"SubCategories": [["SubCateName", "SubCateURL", "SubCateID"]]
}]
std_info = dictfier.dictfy(data, query)
file_name = uuid.uuid4().hex + ".json"
completeName = os.path.join(os.getcwd(), file_name)
file = open(completeName, "w", encoding="utf8")
file.write(json.dumps(std_info))
file.close()
print(completeName)
def get_model_data_not_auth(self, model, **params):
records = request.env[model].search([])
rule_query = records.fields_get_keys()
if model == 'sgu.product':
rule_query = ["id", "name", "price",
"image_url", "type", "on_hand", "color",
"ram", "memory", "origin","vendor", "screen",
"osystem", "camera", "cpu", "pin", "category_id"]
if "query" in params:
query = json.loads(params["query"])
set_query = set(query)
query = [list(set_query.intersection(set(rule_query)))]
else:
query = [rule_query]
if "exclude" in params:
exclude = json.loads(params["exclude"])
for field in exclude:
if field in query[0]:
field_to_exclude= query[0].index(field)
query[0].pop(field_to_exclude)
if "filter" in params:
json_str = params["filter"].replace('"*"', '"&"')
filters = json.loads(json_str)
records = request.env[model].search(filters)
prev_page = None
next_page = None
total_page_number = 1
current_page = 1
if "page_size" in params:
page_size = int(params["page_size"])
count = len(records)
total_page_number = math.ceil(count/page_size)
if "page" in params:
current_page = int(params["page"])
else:
current_page = 1 # Default page Number
start = page_size*(current_page-1)
stop = current_page*page_size
records = records[start:stop]
next_page = current_page+1 \
if 0 < current_page + 1 <= total_page_number \
else None
prev_page = current_page-1 \
if 0 < current_page - 1 <= total_page_number \
else None
if "limit" in params:
limit = int(params["limit"])
records = records[0:limit]
data = dictfier.dictfy(
records,
query,
flat_obj=flat_obj,
nested_flat_obj=nested_flat_obj,
nested_iter_obj=nested_iter_obj
)
res = {
"count": len(records),
"prev": prev_page,
"current": current_page,
"next": next_page,
"total_pages": total_page_number,
"result": data
}
return http.Response(
json.dumps(res),
status=200,
mimetype='application/json'
)
def get_model_data(self, model, **params):
records = request.env[model].search([])
if "query" in params:
query = json.loads(params["query"])
else:
query = [records.fields_get_keys()]
if "exclude" in params:
exclude = json.loads(params["exclude"])
for field in exclude:
if field in query[0]:
field_to_exclude= query[0].index(field)
query[0].pop(field_to_exclude)
if "filter" in params:
json_str = params["filter"].replace('"*"', '"&"')
filters = json.loads(json_str)
records = request.env[model].search(filters)
prev_page = None
next_page = None
total_page_number = 1
current_page = 1
if "page_size" in params:
page_size = int(params["page_size"])
count = len(records)
total_page_number = math.ceil(count/page_size)
if "page" in params:
current_page = int(params["page"])
else:
current_page = 1 # Default page Number
start = page_size*(current_page-1)
stop = current_page*page_size
records = records[start:stop]
next_page = current_page+1 \
if 0 < current_page + 1 <= total_page_number \
else None
prev_page = current_page-1 \
if 0 < current_page - 1 <= total_page_number \
else None
if "limit" in params:
limit = int(params["limit"])
records = records[0:limit]
data = dictfier.dictfy(
records,
query,
flat_obj=flat_obj,
nested_flat_obj=nested_flat_obj,
nested_iter_obj=nested_iter_obj
)
res = {
"count": len(records),
"prev": prev_page,
"current": current_page,
"next": next_page,
"total_pages": total_page_number,
"result": data
}
return http.Response(
json.dumps(res),
status=200,
mimetype='application/json'
)
cate_id_field = "CateID"
list_cate_str = []
for cate in categories_list:
url = cate.a["href"]
cate_name = cate.a.text
cate_id = url.split("/")[-2]
# cate_str = "{\"" + cate_name_field + "\":\"" + cate_name + \
# "\",\"" + cate_id_field + "\":\"" + cate_id + "\"}"
category = Category(cate_name, cate_id)
list_cate_str.append(category)
#cate = "{\"Product\":[" + ",".join(list_str) + "] ,\"Cate\":[" + ",".join(list_cate_str) + "]}"
data = DataHomePage(list_str, list_cate_str)
query = [{
"Products": [["Name", "URL", "Price", "ImgLink"]]
}, {
"Categories": [["CateName", "CateId"]]
}]
std_info = dictfier.dictfy(data, query)
file_name = uuid.uuid4().hex + ".json"
completeName = os.path.join(os.getcwd(), file_name)
file = open(completeName, "w", encoding="utf8")
file.write(json.dumps(std_info))
file.close()
print(completeName)
