def get_model(self,
design_topic_item: DesignTopicItem) -> DesignTopicModel:
design_topic_model = DesignTopicModel()
design_topic_model._id = utils.get_uuid()
design_topic_model.title = design_topic_item['title']
design_topic_model.description = design_topic_item['description']
design_topic_model.html_url = design_topic_item['html_url']
design_topic_model.article = design_topic_item['article']
design_topic_model.create_time = utils.get_utc_time()
return design_topic_model
def get_design_strategy_model(
self,
design_strategy_item: DesignStrategyItem) -> DesignStrategyModel:
design_strategy_model = DesignStrategyModel()
design_strategy_model.id = utils.get_uuid()
design_strategy_model.title = design_strategy_item['title']
design_strategy_model.html_url = design_strategy_item['html_url']
design_strategy_model.description = design_strategy_item['description']
design_strategy_model.content = design_strategy_item['content']
design_strategy_model.category = design_strategy_item['category']
design_strategy_model.create_time = utils.get_utc_time()
return design_strategy_model
def get_design_picture_model(
self,
design_picture_item: DesignPictureItem) -> DesignPictureModel:
design_picture_model = DesignPictureModel()
design_picture_model.id = utils.get_uuid()
design_picture_model.fid = design_picture_item['fid']
design_picture_model.title = design_picture_item['title']
design_picture_model.sub_title = design_picture_item['sub_title']
design_picture_model.html_url = design_picture_item['html_url']
design_picture_model.tags = design_picture_item['tags']
design_picture_model.description = design_picture_item['description']
design_picture_model.img_url = design_picture_item['img_url']
design_picture_model.img_width = design_picture_item['img_width']
design_picture_model.img_height = design_picture_item['img_height']
design_picture_model.img_name = design_picture_item['img_name']
design_picture_model.create_time = utils.get_utc_time()
return design_picture_model
def parse_content(self, response):
uuid = utils.get_uuid()
cid = response.meta['cid']
title = response.meta['title']
try:
data = json.loads(response.text)
except:
print("-----------------------获取到json:" + response.text +
"------------------------------")
return
data_img_list = data['dataImg']
for _data_img in data_img_list:
data_album_list = _data_img['album']
for data_album in data_album_list:
data_img = data_album['l']
# http://pic.to8to.com/case/1605/05/20160505_f0af86a239d0b02e9635a47ih5l1riuq_sp.jpg
img_url = 'http://pic.to8to.com/case/{short_name}'.format(
short_name=data_img['s'])
if self.design_picture_service.is_duplicate_url(img_url):
break
sub_title = data_img['t']
original_width = data_img['w']
original_height = data_img['h']
tags = []
try:
zoom_type = ZONE_TYPE[data_img['zid']]
if zoom_type is not None or not zoom_type.strip() == '':
tags.append(zoom_type)
except KeyError:
pass
try:
style_id = STYLE_ID[data_img['sid']]
if style_id is not None or not style_id.strip() == '':
tags.append(style_id)
except KeyError:
pass
try:
area = AREA[data_img['a']]
if area is not None or not area.strip() == '':
tags.append(area)
except KeyError:
pass
try:
color_id = COLOR_ID[data_img['coid']]
if color_id is not None or not color_id.strip() == '':
tags.append(color_id)
except KeyError:
pass
try:
house_type = HX_ID[data_img['hxid']]
if house_type is not None or not house_type.strip() == '':
tags.append(house_type)
except KeyError:
pass
try:
part = PART_ID[data_img['pid']]
if part is not None or not part.strip() == '':
tags.append(part)
except KeyError:
pass
try:
design_picture_item = DesignPictureItem(
) # type: DesignPictureItem
design_picture_item['fid'] = uuid
design_picture_item['html_url'] = response.url
design_picture_item['img_url'] = img_url
design_picture_item['tags'] = tags
design_picture_item['title'] = title
design_picture_item['sub_title'] = sub_title
design_picture_item['img_width'] = str(original_width)
design_picture_item['img_height'] = str(original_height)
design_picture_item['description'] = design_picture_item[
'title']
yield design_picture_item
except Exception as e:
print("-----------------------获取到json:" + response.text +
"------------------------------")
log.warn("%s ( refer: %s )" % (e, response.url))
if config.USE_PROXY:
proxy_pool.add_failed_time(
response.meta['proxy'].replace('http://', ''))