def __init__(self):
self.dao = CrawlDao.CrawlDao("modeldatas")
self.fileDao = FileUtil.FileDao()
self.stringUtil = StringUtil.StringUtil()
self.sql_category_table = 'db_category'
self.sql_part_table = 'db_category_part'
# 已更新的分类 name+level+parent_id
self.category_have_list = list()
# 已更新的part name+level
self.part_have_list = list()
self.label_dict = {u'字标': '1', u'灯泡': '2', u'四滤': '3', u'': '0'}
self.cat_kind_dict = {u'全车件': 1, u'易损件': 0}
# 分类无法判断标识
self.cat_kind_not = '3'
self.aliss_name_text = ''
self.label_text = ''
# db_category level:dict(name+parent_id)
self.level_name = self.init_category()
# db_category_part name+cate_id:dict(part)
# self.part_name = self.init_part()
self.init_part()
# name+level+parent_id:id
self.category_id_dict = dict()
# name+level+parent_id:vehicle_code
self.category_code_dict = dict()
# cat_id:cat_name
self.cat_id_name = dict()
# 是否是新增的类目
self.is_new = False
def __init__(self):
self.dao = CrawlDao.CrawlDao()
self.fileDao = FileUtil.FileDao()
self.stringUtil = StringUtil.StringUtil()
self.table_mapping = 'db_category_mapping'
self.vehicle_dict = {"商用车": "H", "乘用车": "C"}
# 新分类的
self.cate_dict = dict()
cate_sql = "select cat_id,cat_name,vehicle_code,cat_level,parent_id from db_category_new "
cate_array = self.dao.db.get_data(cate_sql)
for cate_data in cate_array:
cat_id = str(cate_data['cat_id'])
cat_name = str(cate_data['cat_name'])
cat_level = str(cate_data['cat_level'])
vehicle_code = str(cate_data['vehicle_code'])
parent_id = str(cate_data['parent_id'])
key = cat_name + "_" + cat_level + "_" + parent_id
if cat_level == '3':
key += "_" + vehicle_code
self.cate_dict[key] = cat_id
# 老cate的图片数据
self.old_cat_dict = dict()
old_cat_sql = "select cat_name,category_thumb from db_category where cat_id < 2999 and parent_id = 0 and is_deleted = 'N'"
old_cate_array = self.dao.db.get_data(old_cat_sql)
for old_cate_data in old_cate_array:
cat_name = str(old_cate_data['cat_name'])
category_thumb = str(old_cate_data['category_thumb'])
self.old_cat_dict[cat_name] = category_thumb
def main(self, excle):
print '===============start all==================='
# 数据初始化
fileDao = FileUtil.FileDao()
# 单个excle处理
data = fileDao.open_excel(excle)
table = data.sheets()[0]
nrows = table.nrows # 行数
ncols = table.ncols # 列数
print ('行数：%s ,列数：%s' %(nrows , ncols))
# measure_unit_table = data.sheets()[1]
#
# for rownum in range(1, measure_unit_table.nrows):
# row = measure_unit_table.row_values(rownum)
# key = row[0].strip()
# value = row[1].strip()
# self.measure_unit_dic[key] = value
# 第一行
self.first_row_process(table , ncols)
# 第二行开始的后面所有
self.next_row_process(table, nrows, ncols)
print '===============end all==================='
def __init__(self):
self.dao = CrawlDao.CrawlDao()
self.fileDao = FileUtil.FileDao()
self.stringUtil = StringUtil.StringUtil()
# p_id+code : id
self.cate_id_dict = dict()
# self.cate_name_dict = dict()
cate_sql = 'SELECT cat_id,cat_name,cat_code,parent_id,vehicle_code,cat_level FROM db_category_dian WHERE cat_id > 2999 '
cate_array = self.dao.db.get_data(cate_sql)
for cate_data in cate_array:
cat_id = str(cate_data['cat_id'])
cat_code = str(cate_data['cat_code'])
parent_id = str(cate_data['parent_id'])
cat_level = str(cate_data['cat_level'])
vehicle_code = str(cate_data['vehicle_code'])
key = str(parent_id) + "_" + str(cat_code)
if cat_level == '3':
key += "_" + vehicle_code
self.cate_id_dict[key] = cat_id
# self.cate_name_dict[cat_id] = str(cate_data['cat_name'])
# g_id:old_cat_id
self.goods_dict = dict()
goods_sql = "select cat_id,goods_id from db_goods where seller_id = 1"
goods_array = self.dao.db.get_data(goods_sql)
for goods_data in goods_array:
self.goods_dict[str(goods_data['goods_id'])] = str(
goods_data['cat_id'])
def __init__(self):
self.dao = CrawlDao.CrawlDao("modeldatas")
self.fileDao = FileUtil.FileDao()
self.part_code_id_dict = dict()
part_sql = "select id,sum_code from db_category_part where is_deleted = 'N'"
for part_data in self.dao.db.get_data(part_sql):
self.part_code_id_dict[str(part_data['sum_code'])] = int(
part_data['id'])
def __init__(self):
self. fileDao = FileUtil.FileDao()
self.dao = CrawlDao.CrawlDao()
# 获得新老的对应关系
relation_sql_string = 'select my_cat_id,old_cat_id from db_category_relation'
result_array = self.dao.db.get_data(relation_sql_string)
self.relation_table = {}
for relation_result in result_array:
self.relation_table[int(relation_result['old_cat_id'])] = int(relation_result['my_cat_id'])
def __init__(self):
self.dao = CrawlDao.CrawlDao("athena_center", "local")
self.fileDao = FileUtil.FileDao()
self.stringUtil = StringUtil.StringUtil()
self.insert_pic_data_list = list()
self.update_goods_car_data_list = list()
# mac pic
self.max_pic_id = 0
# {{pic_num,pic_index}:[oe,oe...]}
self.pic_dict = dict()
def __init__(self):
self.athena_dao = CrawlDao.CrawlDao("athena", "local")
self.monkey_dao = CrawlDao.CrawlDao('modeldatas', "local")
self.fileDao = FileUtil.FileDao()
self.insert_part_data_list = list()
self.updata_part_data_list = list()
self.monkey_updata_goods_list = list()
# cat_name+paren_id : cat_id
self.athena_cat_dict = dict()
self.init_athena()
def __init__(self):
self.dao = CrawlDao.CrawlDao("test", "local")
self.fileDao = FileUtil.FileDao()
# car_brand+"_"+company:{data}
self.tuhu_car_dict = dict()
# tuhu_car_id:oil_used
self.tuhu_oil_dict = dict()
self.initTuhuCar()
self.initTuhuOil()
self.tuhu_car_dict_keys = self.tuhu_car_dict.keys()
self.tuhu_oil_dict_keys = self.tuhu_oil_dict.keys()
## 缓存
self.online_car_liyang_cache = dict()
def __init__(self):
self.dao = CrawlDao.CrawlDao("modeldatas")
self.fileDao = FileUtil.FileDao()
# 生成的excel
wb = Workbook()
self.write_ew = ExcelWriter(workbook=wb)
self.lop_sheet = wb.create_sheet(u"导入lop数据", 0)
self.wrong_sheet = wb.create_sheet(u"错误数据", 1)
self.init_write_ew()
# 保存的行数
self.save_lop_row_num = 1
self.save_wrong_row_num = 1
# part表的基础信息sum_code:data
self.part_dict = dict()
self.init_part()
# 配件库 oe:list(part_code)
self.part_oe_code_dict = dict()
self.init_part_goods_base()
def __init__(self):
self.dao = CrawlDao.CrawlDao('test', "local")
self.fileDao = FileUtil.FileDao()
self.file_parent = r'/Users/zxg/Desktop/lvqing'
# ===初始化的变量====
# brand_factory:list<map>{leyel_id,series,vehicle_type,model_year,displacement ,intake_style , max_power , fuel_type}
self.liyang_dict_key = "{}_{}_{}"
self.liyang_dict = dict()
# liyang_id:online_car_id
self.liyang_tqmall_dict = dict()
# car_category
# online_car_id_list
self.car_category_list = list()
# id:{brand,series,model,power,year,name}
self.car_category_dict = dict()
self.init_liyang()
self.init_liyang_tqmall()
self.init_car()
# ====处理excel的变量===
# goods_format:goods_size
self.goods_dict = dict()
# goods_format:set(online_car_id)
self.goods_car_dict = dict()
# online_car_id:set(goods_format)
self.car_goods_dict = dict()
# liyang_car_id:set(goods_format)
self.liyang_car_goods_dict = dict()
# liyang_id:{liyang_data}
self.liyang_id_data_dict = dict()
# 不匹配的车型列表
self.not_car_list = copy.deepcopy(self.car_category_list)
# excle中没有匹配上的数据
self.wrong_data_list = list()
def __init__(self):
self.dao = CrawlDao.CrawlDao()
self.fileDao = FileUtil.FileDao()
self.sql_table = 'db_category'
self.sql_relation_table = 'db_category_relation'
self.update_sql_string = set()
def __init__(self):
self.dao = CrawlDao.CrawlDao("modeldatas", "local")
self.fileDao = FileUtil.FileDao()
# {{excle_name:{excel_num:{part_name:"",part_code:"",amount:""}}
self.true_excel_dict = dict()
# encoding=utf-8
__author__ = 'zxg'
from util import CrawlDao, FileUtil
dao = CrawlDao.CrawlDao("test", "local")
fileDao = FileUtil.FileDao()
print "=====start======"
warehouse_id = 22129
org_id = 50450
seller_id = 10155
goods_start_id = 1100000
goods_data_list = list()
third_data_list = list()
# sql
brand_dict = dict()
brand_sql = "select brand_id,brand_name from db_brand"
brand_array = dao.db.get_data(brand_sql)
for brand_data in brand_array:
brand_dict[str(brand_data['brand_name'])] = str(brand_data['brand_id'])
excle_file = r'/Users/zxg/Desktop/Hyundai-杭州韩现配件商品-- 数据2016-5500+.xls'
# 单个excle处理
data = fileDao.open_excel(excle_file)
#
table = data.sheets()[0]
def __init__(self):
self.fileDao = FileUtil.FileDao()
self.dao = CrawlDao.CrawlDao("test", "local")
def __init__(self, *name, **kwargs):
self.dao = CrawlDao.CrawlDao()
# 数据初始化
self.fileDao = FileUtil.FileDao()
def __init__(self):
self.dao = CrawlDao.CrawlDao("test", "local")
self.fileDao = FileUtil.FileDao()
# {{excel_oe:{part_name:"",part_code:"",remarks:""}}
self.true_excel_dict = dict()
def __init__(self, *name, **kwargs):
self.dao = CrawlDao.CrawlDao()
# 数据初始化
self.fileDao = FileUtil.FileDao()
self.update_sql_string = set()
