def save_products(products):
db = dtb()
Product = Products(db)
saved_items = {}
error_items = []
count = 0
total = 0
for product in products:
try:
item = Product.save_from_json(product)
if item["category"]["name"] + "_" + str(
item["category"]["gender"]) not in saved_items.keys():
saved_items[item["category"]["name"] + "_" +
str(item["category"]["gender"])] = 0
saved_items[item["category"]["name"] + "_" +
str(item["category"]["gender"])] += 1
count += 1
except:
raise
error_items.append(product)
logging.error('SAVING PRODUCT: ' + str(json.dumps(product)))
total += 1
print count, len(products)
if total % 50 == 0:
logging.info('SAVING PRODUCTS: ' + str(total) + "/" +
str(len(products)))
return {
"success": saved_items,
"error": error_items,
"error_total": len(error_items),
"success_total": count
}
def load_metadata(website):
db = dtb()
Gender = Genders(db)
genders = Gender.find_all()
Category = Categories(db)
categories = Category.find_all()
Brand = Brands(db)
brands = Brand.find_all()
website = website
products = load_existing_products(db, website)
Size = Sizes(db)
sizes = Size.find_all()
Style = Styles(db)
styles = Style.find_all()
Variant = Variants(db)
variants = Variant.find_all()
return {
"genders": genders,
"categories": categories,
"brands": brands,
"website": website,
"products": products,
"sizes": sizes,
"styles": styles,
"variants": variants
}
def find_products(website):
db = dtb()
Product = Products(db)
conditions = [["website_id", "=", website["id"], "String"]]
if website["name"] == "zalora":
conditions += zalora_conditions
return Product.find_urls_by(conditions)
def save_products(products, meta, product_ids=[]):
db = dtb()
Product = Products(db)
saved_items = {}
error_items = []
count = 0
total = 0
update_queries = []
insert_queries = []
update_discount_queries = []
insert_prices_queries = []
update_price_queries = []
insert_product_sizes_queries = []
insert_images_queries = []
insert_styles_queries = []
clean_data(db, products, meta["website"], meta["products"])
for product in products:
try:
item, meta, update_query, insert_query, update_discount_query, insert_prices, update_price_query, insert_product_sizes, insert_images, insert_styles = Product.save_from_json_3(
product, meta)
update_queries += update_query
insert_queries += insert_query
update_discount_queries += update_discount_query
insert_prices_queries += insert_prices
update_price_queries += update_price_query
insert_product_sizes_queries += insert_product_sizes
insert_images_queries += insert_images
insert_styles_queries += insert_styles
#print insert_images
count += 1
except:
raise
error_items.append(product)
logging.error('SAVING PRODUCT: ' + str(json.dumps(product)))
total += 1
print count, len(products)
if total % 50 == 0:
logging.info('SAVING PRODUCTS: ' + str(total) + "/" +
str(len(products)))
pids, pskus = Product.save_all_3(
update_queries, insert_queries, update_discount_queries,
insert_prices_queries, update_price_queries,
insert_product_sizes_queries, insert_images_queries,
insert_styles_queries)
product_ids += pids
for k in pskus.keys():
if k not in meta["products"].keys():
meta["products"][k] = pskus[k]
db.commit()
return {
"success": saved_items,
"error": error_items,
"error_total": len(error_items),
"success_total": count
}, product_ids, meta
def find_website(name):
db = dtb()
Website = Websites(db)
website = Website.find_by_name(name.strip())
if not website:
website = Website.save(name.strip())
db.commit()
if not website:
return None
return website
def delete_products(product_ids, website):
db = dtb()
Product = Products(db)
Product.delete_all(product_ids, website)
db.commit()
def load_data(path):
return json.load(open(path))
def load_variants(db):
Variant = Variants(db)
variants = Variant.find_all()
return variants
if __name__ == "__main__":
logging.info('STARTING: listing files in ' + sys.argv[1])
files = list_files(sys.argv[1])
logging.info('LISTING FILES: ' + str(len(files)))
files = [files[-1]]
db = dtb()
Product = Products(db)
variants = load_variants(db)
for f in files:
logging.info('READING FILE: ' + str(f))
print 'READING FILE: ' + str(f)
products = load_data(f)
counter = 0
for prod in products:
description = []
if "description" in prod.keys():
description = prod["description"]
else:
description = prod["features"]
Product.update_description_2(prod["id"], description)
# if "images" in prod.keys():
def save_products(products, meta, product_ids=[]):
db = dtb()
Product = Products(db)
saved_items = {}
error_items = []
count = 0
total = 0
update_queries = []
insert_queries = []
update_discount_queries = []
insert_prices_queries = []
update_price_queries = []
update_size_availability_queries = []
insert_product_sizes_queries = []
delete_missing_sizes_queries = []
insert_images_queries = []
delete_images_queries = []
insert_styles_queries = []
delete_styles_queries = []
for product in products:
try:
item, meta, update_query, insert_query, update_discount_query, insert_prices, update_price_query, update_size_availability, insert_product_sizes, delete_missing_sizes, insert_images, delete_images, insert_styles, delete_styles = Product.save_from_json_2(
product, meta)
update_queries += update_query
insert_queries += insert_query
update_discount_queries += update_discount_query
insert_prices_queries += insert_prices
update_price_queries += update_price_query
update_size_availability_queries += update_size_availability
insert_product_sizes_queries += insert_product_sizes
delete_missing_sizes_queries += delete_missing_sizes
insert_images_queries += insert_images
delete_images_queries += delete_images
insert_styles_queries += insert_styles
delete_styles_queries += delete_styles
#print insert_images
count += 1
except:
raise
error_items.append(product)
logging.error('SAVING PRODUCT: ' + str(json.dumps(product)))
total += 1
print count, len(products)
if total % 50 == 0:
logging.info('SAVING PRODUCTS: ' + str(total) + "/" +
str(len(products)))
product_ids += Product.save_all(
update_queries, insert_queries, update_discount_queries,
insert_prices_queries, update_price_queries,
update_size_availability_queries, insert_product_sizes_queries,
delete_missing_sizes_queries, insert_images_queries,
delete_images_queries, insert_styles_queries, delete_styles_queries)
db.commit()
return {
"success": saved_items,
"error": error_items,
"error_total": len(error_items),
"success_total": count
}, product_ids
def find_website(name):
db = dtb()
Website = Websites(db)
return Website.find_by_name(name)