def login_mobile(self) -> bool:
# 访问移动端
self.driver.get('http://m.ac.qq.com/')
# 点击右上角菜单
self.driver.find_element_by_css_selector(
'body > header > a.btn-top.menu').click()
# 点击我的地盘
self.driver.find_element_by_css_selector(
'body > div.nav-menu-box.show > div.nav-menu > a').click()
if get_current_url() == 'http://m.ac.qq.com/home/index':
logger.info('已经登录')
return True
# 输入用户名
self.driver.find_element_by_css_selector('#u').send_keys(
data[DATA_USERNAME])
self.driver.find_element_by_css_selector('#p').send_keys(
data[DATA_PASSWORD])
time.sleep(1)
self.driver.find_element_by_css_selector('#go').click()
time.sleep(5)
ok = get_current_url() == 'http://m.ac.qq.com/home/index'
status = PLATFORM_STATUS_AUTH_OK if ok else PLATFORM_STATUS_AUTH_FAIL
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
return ok
def login(self) -> bool:
login_url = get_current_url()
self.driver.switch_to.frame('login_ifr')
self.driver.find_element_by_css_selector("#switcher_plogin").click()
clear_and_send_keys("#u", data[DATA_USERNAME])
time.sleep(1)
clear_and_send_keys("#p", data[DATA_PASSWORD])
time.sleep(2)
self.driver.find_element_by_css_selector("#login_button").click()
time.sleep(3)
if get_current_url() != AUTH_OK_URL:
logger.info(get_current_url())
status = PLATFORM_STATUS_AUTH_FAIL
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
raise Pang5Exception("登录异常")
ok = get_current_url() != login_url
status = PLATFORM_STATUS_AUTH_OK if ok else PLATFORM_STATUS_AUTH_FAIL
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
return ok
def handle_go_link(self, item):
if item.has_key('crawl_go_link'):
url = item['crawl_go_link']
if not utils.has_prefix(url, 'http://'):
info = 'warning ignore ' + url
logger.debug(info)
item['not_access_reason'] = info
return STAT_NOT_ACCESS
if utils.get_url_by_browser_for_times(self.driver, url, 3):
time.sleep(1)
current_url = utils.get_current_url(self.driver)
if not current_url:
info = 'failed to get ' + url
logger.debug(info)
item['not_access_reason'] = info
return STAT_NOT_ACCESS
info = str(item['crawl_go_link'])
item['go_link'] = current_url
info = info + ' -> ' + str(item['go_link'])
logger.debug('change ' + info)
return STAT_CHANGE
else:
info = 'failed to get url ' + url
logger.debug(info)
item['not_access_reason'] = info
return STAT_NOT_ACCESS
else:
return STAT_NOT_CHANGE
def qq_login(self, driver, login_username, login_password) -> bool:
logger.info('用qq登录')
get('https://manhua.163.com/')
# click('.topbar-meta-user >ul >li:nth-child(1)>.js-login-required')
driver.find_element_by_css_selector(
'.topbar-meta-user >ul >li:nth-child(1)>.js-login-required').click(
)
time.sleep(3)
driver.find_element_by_css_selector(
'#common_login > div.m-loginswitch > ul > li:nth-child(2) > a > i'
).click()
time.sleep(3)
print(get_current_url())
windows = driver.window_handles
driver.switch_to.window(windows[-1])
driver.switch_to_frame(driver.find_element_by_id("ptlogin_iframe"))
driver.find_element_by_id('switcher_plogin').click()
time.sleep(3)
u = driver.find_element_by_id('u')
u.clear()
u.send_keys(login_username)
p = driver.find_element_by_id('p')
p.clear()
p.send_keys(login_password)
driver.find_element_by_id('login_button').click()
time.sleep(3)
windows = driver.window_handles
driver.switch_to.window(windows[-1])
return True
def weibo_login(self, driver):
logger.info('用微博登录')
get('https://h5.manhua.163.com/login/form?targetUrl=https%3A%2F%2Fh5.manhua.163.com%2Fsubscribe%3Fsigned_in_callback%3D1'
)
driver.find_element_by_css_selector('#sina').click()
time.sleep(3)
# 一个字母一个字母的输入
# for i in list(data[DATA_USERNAME]):
# driver.find_element_by_css_selector('#userId').send_keys(i)
# time.sleep(0.5)
# time.sleep(2)
# for i in list(data[DATA_PASSWORD]):
# driver.find_element_by_css_selector('#passwd').send_keys(i)
# time.sleep(0.5)
driver.find_element_by_css_selector('#userId').send_keys(
data[DATA_USERNAME])
time.sleep(0.5)
driver.find_element_by_css_selector('#passwd').send_keys(
data[DATA_PASSWORD])
time.sleep(0.5)
# js = f'document.getElementById("userId").setAttribute("value","{data[DATA_USERNAME]}");' \
# f'document.getElementById("passwd").setAttribute("value","{data[DATA_PASSWORD]}");'
# driver.execute_script(js)
# import pyautogui
# pyautogui.press('f5')
# time.sleep(2)
# js = 'document.getElementById("userId").focus()'
# driver.execute_script(js)
# pyautogui.typewrite(data[DATA_USERNAME])
# time.sleep(2)
# pyautogui.keyDown('tab')
# pyautogui.keyUp('tab')
# pyautogui.typewrite(data[DATA_PASSWORD])
# time.sleep(2)
# 处理验证码问题
# 火狐浏览器的微博登录是pc端的,所以样式都要用pc端的.
# 为什么必须用火狐, 因为只有火狐能对元素截图
captcha_div = driver.find_element_by_css_selector(
'p.oauth_code:nth-child(3)')
if captcha_div.is_displayed():
logger.info('有验证码, 截图')
captcha_img = captcha_div.find_element_by_css_selector('span img')
store_path = f"{CAPTCHAR_PATH}/netEase_{datetime.now().strftime('%Y-%m-%d %H%M%S')}.png"
captcha_img.screenshot(store_path)
logger.info('调用服务识别')
result, ok = image_recog(store_path)
if ok:
logger.info(f'识别成功,填写{result}')
captcha_div.find_element_by_css_selector('input').send_keys(
result)
else:
raise Pang5Exception('验证码识别失败, 请重试')
driver.find_element_by_css_selector('a.WB_btn_login').click()
# pyautogui.press('enter')
time.sleep(3)
return 'weibo' in get_current_url()
def process(self, mysql_id):
g_mysqlid["mysql_id"] = mysql_id
with open_driver(browser='firefox') as driver:
with track_alert(driver):
self.driver = driver
# 处理登录
get(MANAGE_URL)
if get_current_url() != MANAGE_URL:
if not self.login():
raise Pang5Exception('登录失败')
logger.info('登录成功')
# 根据作品名称点击对应的新建章节
items = driver.find_elements_by_css_selector(
'#submission > div.container > ul > li')
for item in items:
work_name = item.find_element_by_css_selector(
'div.content > h3').text
logger.debug(f'发现作品{work_name}')
if work_name.strip() == data[DATA_WORKS_NAME].strip():
new_chapter = item.find_element_by_css_selector(
'nav > a:nth-child(2)')
new_chapter.click()
time.sleep(5)
handles = driver.window_handles
if len(handles) == 1:
logger.error(
f'点击创建章节失败 maimeng_series={data[DATA_WORKS_NAME]}')
return
else:
driver.switch_to_window(handles[1])
self.publish()
def login_mobile_qq(self):
self.driver.get(LOGIN_MOBILE_URL)
# 点击qq
self.driver.find_element_by_css_selector(
'#coagent > a.coagent.coagent-qq').click()
time.sleep(1)
self.driver.switch_to_frame(
self.driver.find_element_by_id("ptlogin_iframe"))
self.driver.find_element_by_id('switcher_plogin').click()
time.sleep(3)
u = self.driver.find_element_by_id('u')
u.clear()
u.send_keys(data[DATA_USERNAME])
p = self.driver.find_element_by_id('p')
p.clear()
p.send_keys(data[DATA_PASSWORD])
self.driver.find_element_by_id('login_button').click()
time.sleep(3)
ok = get_current_url() == AUTH_MOBILE_OK_URL
status = PLATFORM_STATUS_AUTH_OK if ok else PLATFORM_STATUS_AUTH_FAIL
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
return ok
def login(self) -> bool:
login_url = get_current_url()
js = f'''$("#login_username").val("{data[DATA_USERNAME]}"); $("#login_pwd").val("{data[DATA_PASSWORD]}"); $("a.login_btn:nth-child(4)").click();'''
self.driver.execute_script(js)
time.sleep(3)
if get_current_url() != AUTH_OK_URL:
status = PLATFORM_STATUS_AUTH_FAIL
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
raise Pang5Exception("登录失败")
status = PLATFORM_STATUS_AUTH_OK
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
return True
def login(self) -> bool:
login_url = get_current_url()
clear_and_send_keys(".username-field > input:nth-child(2)",
data[DATA_USERNAME])
clear_and_send_keys(".password-field > input:nth-child(2)",
data[DATA_PASSWORD])
time.sleep(3)
self.driver.find_element_by_css_selector(".login-btn").click()
time.sleep(3)
if get_current_url() != AUTH_OK_URL:
logger.error(get_current_url())
status = PLATFORM_STATUS_AUTH_FAIL
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
raise Pang5Exception("登录异常")
status = PLATFORM_STATUS_AUTH_OK
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
return True
def process(self, mysql_id):
g_mysqlid["mysql_id"] = mysql_id
with open_driver(browser='firefox') as driver:
with track_alert(driver):
self.driver = driver
get(AUTH_OK_URL)
if get_current_url() != AUTH_OK_URL:
# 登录方式
login_type = data[DATA_LOGIN_TYPE]
if login_type in ('', 'mobile', 'email', 'mail',
'username'):
login = self.login_mobile
elif login_type == 'qq':
login = self.login_mobile_qq
elif login_type == 'weibo':
login = self.login_mobile_weibo
else:
raise Pang5Exception(f'登录方式不支持{login_type}')
if not login():
status = PLATFORM_STATUS_AUTH_FAIL
update_login_status(
platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
raise Pang5Exception("登录失败")
logger.info('登录成功')
logger.info('点击新建章节')
new_chapter_url = f'http://comic.user.u17.com/chapter/chapter_add.php?comic_id={data[DATA_THIRD_ID]}'
self.driver.get(new_chapter_url)
time.sleep(3)
if get_current_url() != new_chapter_url:
raise Pang5Exception(f'有妖气作品id({data[DATA_THIRD_ID]})有误')
self.publish()
def login_mobile(self) -> bool:
self.driver.get(LOGIN_MOBILE_URL)
self.driver.find_element_by_css_selector(
'#wrapper > div > div:nth-child(2) > input').send_keys(
data[DATA_USERNAME])
self.driver.find_element_by_css_selector(
'#wrapper > div > div:nth-child(3) > input:nth-child(1)'
).send_keys(data[DATA_PASSWORD])
time.sleep(1)
self.driver.find_element_by_css_selector(
'#wrapper > div > a.green-btn.login-btn').click()
time.sleep(2)
ok = get_current_url() == AUTH_MOBILE_OK_URL
status = PLATFORM_STATUS_AUTH_OK if ok else PLATFORM_STATUS_AUTH_FAIL
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
return ok
def login_mobile_weibo(self):
self.driver.get(LOGIN_MOBILE_URL)
# 点击微博
self.driver.find_element_by_css_selector(
'a.coagent-weibo:nth-child(2)').click()
time.sleep(1)
self.driver.find_element_by_css_selector('#userId').send_keys(
data[DATA_USERNAME])
self.driver.find_element_by_css_selector('#passwd').send_keys(
data[DATA_PASSWORD])
time.sleep(1)
self.driver.find_element_by_css_selector('.WB_btn_login').click()
time.sleep(3)
ok = get_current_url() == AUTH_MOBILE_OK_URL
status = PLATFORM_STATUS_AUTH_OK if ok else PLATFORM_STATUS_AUTH_FAIL
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
return ok
def process(self, mysql_id):
g_mysqlid["mysql_id"] = mysql_id
with open_driver() as driver:
self.driver = driver
with track_alert(driver):
LOGIN_USERNAME = data[DATA_USERNAME]
LOGIN_PASSWORD = data[DATA_PASSWORD]
logger.info(f'用户名{LOGIN_USERNAME}')
logger.info(f'密码{LOGIN_PASSWORD}')
# 处理登录
if not self.mobile_login(driver, LOGIN_USERNAME,
LOGIN_PASSWORD):
status = PLATFORM_STATUS_AUTH_FAIL
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
raise Pang5Exception('登录失败')
get(MANAGE_URL)
time.sleep(2)
cur = get_current_url()
if cur != MANAGE_URL:
logger.error(MANAGE_URL)
status = PLATFORM_STATUS_AUTH_FAIL
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
raise Pang5Exception("登录失败")
logger.info('登录成功')
status = PLATFORM_STATUS_AUTH_OK
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
self.driver.find_element_by_link_text('我的作品').click()
self.search_article(data[DATA_WORKS_NAME])
self.form(driver)
def login_operation(dr):
#clicking on accept cookies
dr.find_element_by_xpath(read_xpath("login_user","ElementCookieAccept")).click()
#padding
sleep(paddingtime)
#filling username
dr.find_element_by_xpath(read_xpath("login_user","ElementLoginUsername")).send_keys(username)
#padding
sleep(paddingtime)
#filling password
dr.find_element_by_xpath(read_xpath("login_user","ElementLoginPassword")).send_keys(password)
#padding
sleep(paddingtime)
dr.find_element_by_xpath(read_xpath("login_user","ElementLoginLoginButton")).click()
wait_until_load(dr, "https://www.instagram.com/accounts/onetap/?next=%2F")
sleep(paddingtime)
dr.find_element_by_xpath(read_xpath("login_user","ElementSaveLoginInfoDecline")).click()
sleep(paddingtime)
dr.find_element_by_xpath(read_xpath("homescreen_operations","ElementTurnOnNotificationsDecline")).click()
#just debuging if we are logged in
if get_current_url(dr) == "https://www.instagram.com/":
print("-----Successful login------")
def publish(self):
# 让网站允许Flash
# use_flash()
# 进入上传章节页面
if not FIRST_CHAPTER:
# 有了第一章之后才会出来是否定时发布和发布日期,请提前发布好第一章
if data[DATA_IS_CLOCK] == False:
# 定时发布选否
self.driver.find_element_by_css_selector(
'table > tbody > tr:nth-child(2) > td.chapter-publish-time > label:nth-child(2) > input[type="radio"]'
).click()
else:
# 发布日期
try:
# 定时发布选是
self.driver.find_element_by_css_selector(
'table > tbody > tr:nth-child(2) > td.chapter-publish-time > label:nth-child(1) > input[type="radio"]'
).click()
self.driver.find_element_by_css_selector(
"#chapter_date").send_keys(
data[DATA_CLOCK_PUBLISH_DATETIME])
except NoSuchElementException:
raise Pang5Exception('定时发布时间页面上处于不能选状态')
# 章节名称
clear_and_send_keys("#chapter_title", data[DATA_CHAPTER_NAME])
# 确定修改
self.driver.find_element_by_css_selector("#chapterTitleSubmit").click()
# 章节封面
tips_chapter = data[DATA_WORKS_IMAGE]
logger.info(tips_chapter)
try:
self.driver.find_element_by_css_selector("#Filedata").send_keys(
tips_chapter)
# 点击上传封面
time.sleep(3)
self.driver.find_element_by_css_selector('#btn_upload').click()
except:
logger.info('页面上没有章节封面')
time.sleep(3)
# 上传章节内容
scroll_to()
# self.driver.execute_script('document.querySelectorAll("#button_mid")[0].style.display="block";')
# time.sleep(1)
# 点击上传按钮
# d = self.driver.find_element_by_css_selector("#create_chapter_tip").location_once_scrolled_into_view
# printt(d['x'],d['y'])
click_by_pyautogui(CHAPTER_PNG)
# click_by_pg(*POSOTION_GREEN_BUTTON)
img: str = ' '.join(data[DATA_CHAPTER_IMAGE])
cmd = f'D:/uploadImg.exe 打开 {img}'
logger.info(cmd)
os.system(cmd)
js = 'return $("#uploadProgressBox").text();'
while True:
percent = self.driver.execute_script(js)
time.sleep(4)
logger.info(percent)
if percent == '100%':
break
if REAL_PUBLISH:
create_url = get_current_url()
self.driver.find_element_by_css_selector('#submit_data').click()
time.sleep(2)
if create_url == get_current_url():
logger.error(f'发布失败')
err_msg_eles = self.driver.find_elements_by_css_selector(
"#cover_msg > div")
if len(err_msg_eles) > 0:
err_msg = err_msg_eles[0].text
logger.error(err_msg)
raise Pang5Exception('发布失败，稍后会重试')
logger.info('发布成功')
def process(self, mysql_id):
g_mysqlid["mysql_id"] = mysql_id
with open_driver(phone_ua=True, browser='firefox') as driver:
with track_alert(driver):
# 处理登录
login_username = data[DATA_USERNAME]
login_password = data[DATA_PASSWORD]
logger.info(f'用户名{login_username}')
logger.info(f'密码{login_password}')
get(MANAGE_URL)
if get_current_url() != MANAGE_URL:
if data[DATA_LOGIN_TYPE] == 'mobile':
self.mobile_login(driver, login_username,
login_password)
elif data[DATA_LOGIN_TYPE] in ('mail', '', 'email'):
self.mail_login(driver, login_username, login_password)
elif data[DATA_LOGIN_TYPE] == 'qq':
self.qq_login(driver, login_username, login_password)
elif data[DATA_LOGIN_TYPE] == 'weibo':
self.weibo_login(driver)
elif data[DATA_LOGIN_TYPE] == 'weixin':
self.weixin_login(driver)
else:
raise Pang5Exception('暂时不支持您的登录方式')
# self.mobile_login(driver)
# 登录
# 继续中间页面
logger.info('点击完了登录')
get('https://zz.manhua.163.com/')
time.sleep(3)
if get_current_url() not in [
'http://zz.manhua.163.com/',
'https://zz.manhua.163.com/'
]:
status = PLATFORM_STATUS_AUTH_FAIL
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
raise Pang5Exception('登录失败')
time.sleep(1)
logger.info('登录成功')
status = PLATFORM_STATUS_AUTH_OK
update_login_status(platform=data[DATA_PLATFORM],
platform_username=data[DATA_USERNAME],
platform_password=data[DATA_PASSWORD],
platform_status=status)
# try:
net_series = driver.find_elements_by_link_text(
data[DATA_WORKS_NAME])
if len(net_series) == 0:
raise Pang5Exception(f"该用户下没有作品{data[DATA_WORKS_NAME]}")
net_series[0].click()
time.sleep(1)
handles = driver.window_handles
time.sleep(1)
driver.switch_to_window(handles[-1])
js = "window.scrollTo(0, document.body.scrollHeight)"
driver.execute_script(js)
driver.find_element_by_css_selector(
'#panel1 > div > div > a').click()
self.form(driver)
def parse(self, response):
'''
file = open("a.html", 'w')
file.write(response._body)
file.close()
return
'''
ret_items = []
next_url = self.get_next_page(response)
if next_url:
ret_items.append(Request(url=next_url,dont_filter=True,callback=self.parse))
log.msg('url ' + response._url, level = log.DEBUG)
#return ret_items
hxs = HtmlXPathSelector(response)
h_div_array = hxs.select('//body/div[@class="area"]/div[@class="dealbox"]/div')
i = 0
for h_div in h_div_array:
xpath_list = [
['login_url', 'div/p/a/@href', 'string', None],
['img_url', 'div/p/a/img/@src', 'string', None],
['origin_img_url', 'div/p/a/img/@data-original', 'string', None],
['title', 'div/h2/a/text()', 'string', None],
['current_price_yuan_str', 'div/h4/span/em/text()', 'string', None],
['current_price_fen_str', 'div/h4/span/em/em/text()', 'string', ""],
['origin_price', 'div/h4/span/i/text()', 'get_float_str_to_fen', None],
['start_time_str', 'div/h5/span/text()', 'string', None],
]
attr_dict = get_attr(xpath_list, h_div)
if not attr_dict:
continue
start_time = get_datetime(attr_dict['start_time_str'])
if start_time < utils.get_default_start_time():
log.msg('skip too old time ', level = log.DEBUG)
continue
current_price = get_float_str_to_fen(attr_dict['current_price_yuan_str'] + attr_dict['current_price_fen_str'])
i = i + 1
log.msg('opentab count ' + str(i), level = log.DEBUG)
if i % 5 == 0:
self.recreate_driver()
log.msg('recreate_driver i ' + str(i), level = log.DEBUG)
if not utils.get_url_by_browser(self.driver, attr_dict['login_url']):
continue
log.msg('after get_url_by_browser ', level = log.DEBUG)
#a_obj = self.driver.find_element_by_xpath('//body/div[@id="dialog_out_weldeal"]/div[@class="diginfo"]/div[@class="weloutdialog"]/div[@id="ppLogin"]/form[@name="loginform"]/ul/li[@class="reg"]/a')
a_obj = utils.find_element_by_xpath(self.driver, '//body/div[@id="dialog_out_weldeal"]/div[@class="diginfo"]/div[@class="weloutdialog"]/div[@id="ppLogin"]/form[@name="loginform"]/ul/li[@class="reg"]/a')
log.msg('after find_element_by_xpath ', level = log.DEBUG)
if not a_obj:
log.msg('failed to get url from login_url ' + attr_dict['login_url'], level = log.WARNING)
continue
a_obj.click()
log.msg('after click ', level = log.DEBUG)
#time.sleep(2)
#url = self.driver.current_url
origin_url = utils.get_current_url(self.driver)
if not origin_url:
continue
if origin_url.find('http://s.click.taobao.com') != -1 :
log.msg('skip invalid url ' + origin_url, level = log.DEBUG)
continue
url = origin_url.split('&')[0]
#url = 'http://www.example.com'
log.msg('after current_url ' + url, level = log.DEBUG)
pic_url, item_url, baoyou, cid = utils.get_taobao_item_info(utils.get_id(url))
if not item_url:
log.msg('failed to get item info url ' + url, level = log.DEBUG)
continue
origin_category_name, category_name = self.cg.get_cid_name(cid)
log.msg('origin_category_name ' + origin_category_name + ' category_name ' + category_name + ' title ' + attr_dict['title'] + ' url ' + url, level = log.DEBUG)
discount = get_discount(current_price, attr_dict['origin_price'])
self.log("discount " + str(discount), level = log.DEBUG)
if attr_dict.has_key('origin_img_url') and attr_dict['origin_img_url'][-4:] == '.jpg':
img_url = attr_dict['origin_img_url']
elif attr_dict.has_key('img_url') and attr_dict['img_url'][-4:] == '.jpg':
img_url = attr_dict['img_url']
else:
log.msg('skip invalid img_url ' + attr_dict['img_url'], level = log.WARNING)
continue
prod = Zhe800BaoyouItem()
prod['link'] = url
prod['id'] = hashlib.md5(prod['link']).hexdigest().upper()
prod['title'] = attr_dict['title']
prod['img'] = img_url
prod['ori_price'] = attr_dict['origin_price']
prod['cur_price'] = current_price
prod['discount'] = discount
prod['stat'] = utils.BEGIN
prod['sale'] = UNKNOWN_NUM
prod['sale_percent'] = UNKNOWN_NUM
prod['display_time_begin'] = start_time
prod['display_time_end'] = utils.get_default_end_time()
#prod['display_time_end'] = start_time
#prod['actual_time_begin'] = start_time
#prod['actual_time_end'] = start_time
prod['limit'] = UNLIMITED_NUM
prod['source'] = self.display_name
prod['origin_category_name'] = origin_category_name
prod['category_name'] = category_name
ret_items.append(prod)
if debug :
break
return ret_items
