def ajax_recommentation_in():
date = request.args.get('date', '') # '2013-09-01'
results = get_attr(date)
if results:
return json.dumps(results)
else:
return None
def ajax_recommentation_in():
date = request.args.get('date', '') # '2013-09-01'
results = get_attr(date)
if results:
return json.dumps(results)
else:
return None
def execute_cmd(cmd):
try:
manage_user()
method = get_attr(cmd)
# check if method is whitelisted
if frappe.session['user'] == 'Guest' and (method
not in frappe.guest_methods):
return get_response(message="Not Allowed",
args={"http_status_code": 403})
elif not method in frappe.whitelisted:
return get_response(message="Not Allowed",
args={"http_status_code": 403})
else:
args = get_json_request(frappe.local.form_dict.args)
result = frappe.call(method, args)
if result:
if isinstance(result, dict):
return get_response(message="Success",
status_code=1,
args=result)
else:
return get_response(message="Success", status_code=1)
else:
return get_response(
message="Error occured, Please contact administrator")
except Exception, e:
raise e
def ajax_show_all():
date = request.args.get('date', '') # '2013-09-01'
date = date.replace('-', '')
results = get_attr(date)
if results:
return json.dumps(results)
else:
return None
def ajax_show_all():
date = request.args.get('date', '') # '2013-09-01'
date = date.replace('-', '')
results = get_attr(date)
if results:
return json.dumps(results)
else:
return None
def validate_url():
parts = frappe.request.path[1:].split("/")
call = cmd = None
if len(parts) > 1:
call = parts[1]
if len(parts) > 2:
cmd = parts[2]
if call == "login" and len(parts) == 2:
frappe.local.form_dict.cmd = "login"
elif call == "issue" and len(parts) == 3:
try:
cmd = "helpdesk.helpdesk_api.{0}".format(cmd)
method = get_attr(cmd)
frappe.local.form_dict.cmd = cmd
except Exception, e:
raise Exception("Invalid API-URL")
def execute_cmd(cmd):
try:
manage_user()
method = get_attr(cmd)
# check if method is whitelisted
if frappe.session['user'] == 'Guest' and (method not in frappe.guest_methods):
return get_response(message="Not Allowed", args={"http_status_code":403})
elif not method in frappe.whitelisted:
return get_response(message="Not Allowed", args={"http_status_code":403})
else:
args = get_json_request(frappe.local.form_dict.args)
result = frappe.call(method, args)
if result:
if isinstance(result, dict):
return get_response(message="Success", status_code=1, args=result)
else:
return get_response(message="Success", status_code=1)
else:
return get_response(message="Error occured, Please contact administrator")
except Exception, e:
raise e
def validate_url():
parts = frappe.request.path[1:].split("/")
call = cmd = None
if len(parts) > 1:
call = parts[1]
if len(parts) > 2:
cmd = parts[2]
# if call == "login" and len(parts) == 2:
# frappe.local.form_dict.cmd = "login"
if call in ["login", "logout"] and len(parts) == 2:
frappe.local.form_dict.cmd = call
elif call == "issue" and len(parts) == 3:
try:
cmd = "helpdesk.helpdesk_api.{0}".format(cmd)
method = get_attr(cmd)
frappe.local.form_dict.cmd = cmd
except Exception, e:
raise Exception("Invalid API-URL")
def parse(self, response):
'''
file = open("a.html", 'w')
file.write(response._body)
file.close()
return
'''
ret_items = []
next_url = self.get_next_page(response)
if next_url:
ret_items.append(Request(url=next_url,dont_filter=True,callback=self.parse))
log.msg('url ' + response._url, level = log.DEBUG)
#return ret_items
hxs = HtmlXPathSelector(response)
h_div_array = hxs.select('//body/div[@class="area"]/div[@class="dealbox"]/div')
i = 0
for h_div in h_div_array:
xpath_list = [
['login_url', 'div/p/a/@href', 'string', None],
['img_url', 'div/p/a/img/@src', 'string', None],
['origin_img_url', 'div/p/a/img/@data-original', 'string', None],
['title', 'div/h2/a/text()', 'string', None],
['current_price_yuan_str', 'div/h4/span/em/text()', 'string', None],
['current_price_fen_str', 'div/h4/span/em/em/text()', 'string', ""],
['origin_price', 'div/h4/span/i/text()', 'get_float_str_to_fen', None],
['start_time_str', 'div/h5/span/text()', 'string', None],
]
attr_dict = get_attr(xpath_list, h_div)
if not attr_dict:
continue
start_time = get_datetime(attr_dict['start_time_str'])
if start_time < utils.get_default_start_time():
log.msg('skip too old time ', level = log.DEBUG)
continue
current_price = get_float_str_to_fen(attr_dict['current_price_yuan_str'] + attr_dict['current_price_fen_str'])
i = i + 1
log.msg('opentab count ' + str(i), level = log.DEBUG)
if i % 5 == 0:
self.recreate_driver()
log.msg('recreate_driver i ' + str(i), level = log.DEBUG)
if not utils.get_url_by_browser(self.driver, attr_dict['login_url']):
continue
log.msg('after get_url_by_browser ', level = log.DEBUG)
#a_obj = self.driver.find_element_by_xpath('//body/div[@id="dialog_out_weldeal"]/div[@class="diginfo"]/div[@class="weloutdialog"]/div[@id="ppLogin"]/form[@name="loginform"]/ul/li[@class="reg"]/a')
a_obj = utils.find_element_by_xpath(self.driver, '//body/div[@id="dialog_out_weldeal"]/div[@class="diginfo"]/div[@class="weloutdialog"]/div[@id="ppLogin"]/form[@name="loginform"]/ul/li[@class="reg"]/a')
log.msg('after find_element_by_xpath ', level = log.DEBUG)
if not a_obj:
log.msg('failed to get url from login_url ' + attr_dict['login_url'], level = log.WARNING)
continue
a_obj.click()
log.msg('after click ', level = log.DEBUG)
#time.sleep(2)
#url = self.driver.current_url
origin_url = utils.get_current_url(self.driver)
if not origin_url:
continue
if origin_url.find('http://s.click.taobao.com') != -1 :
log.msg('skip invalid url ' + origin_url, level = log.DEBUG)
continue
url = origin_url.split('&')[0]
#url = 'http://www.example.com'
log.msg('after current_url ' + url, level = log.DEBUG)
pic_url, item_url, baoyou, cid = utils.get_taobao_item_info(utils.get_id(url))
if not item_url:
log.msg('failed to get item info url ' + url, level = log.DEBUG)
continue
origin_category_name, category_name = self.cg.get_cid_name(cid)
log.msg('origin_category_name ' + origin_category_name + ' category_name ' + category_name + ' title ' + attr_dict['title'] + ' url ' + url, level = log.DEBUG)
discount = get_discount(current_price, attr_dict['origin_price'])
self.log("discount " + str(discount), level = log.DEBUG)
if attr_dict.has_key('origin_img_url') and attr_dict['origin_img_url'][-4:] == '.jpg':
img_url = attr_dict['origin_img_url']
elif attr_dict.has_key('img_url') and attr_dict['img_url'][-4:] == '.jpg':
img_url = attr_dict['img_url']
else:
log.msg('skip invalid img_url ' + attr_dict['img_url'], level = log.WARNING)
continue
prod = Zhe800BaoyouItem()
prod['link'] = url
prod['id'] = hashlib.md5(prod['link']).hexdigest().upper()
prod['title'] = attr_dict['title']
prod['img'] = img_url
prod['ori_price'] = attr_dict['origin_price']
prod['cur_price'] = current_price
prod['discount'] = discount
prod['stat'] = utils.BEGIN
prod['sale'] = UNKNOWN_NUM
prod['sale_percent'] = UNKNOWN_NUM
prod['display_time_begin'] = start_time
prod['display_time_end'] = utils.get_default_end_time()
#prod['display_time_end'] = start_time
#prod['actual_time_begin'] = start_time
#prod['actual_time_end'] = start_time
prod['limit'] = UNLIMITED_NUM
prod['source'] = self.display_name
prod['origin_category_name'] = origin_category_name
prod['category_name'] = category_name
ret_items.append(prod)
if debug :
break
return ret_items
def parse(self, response):
hxs = HtmlXPathSelector(response)
h_div = get_one(hxs.select('//body/div[@id="bodydiv"]/div[@class="main_content"]/div/div[@class="left_cont"]/div[@id="msnew"]/div[@class="msnew_infobar"]'))
xpath_list_div = [
['start_time_str', 'div[@class="msnew_infotitle"]/p[@class="msinfotitle_left02"]/text()', 'strip', None],
['from_time', 'span[@class="CountDown"]/@fr', 'int', None],
['to_time', 'span[@class="CountDown"]/@to', 'int', None],
]
if not h_div:
self.log('no page to parse', level = log.WARNING)
return
attr_dict = get_attr(xpath_list_div, h_div)
start_time_str = str(datetime.datetime.now().date()) + " " + attr_dict['start_time_str']
print 'start_time_str ' + start_time_str.encode('utf8')
display_time_begin = int(datetime.datetime.strptime(start_time_str, "%Y-%m-%d %H:%M:%S").strftime("%s"))
print 'display_time_begin ' + str(display_time_begin)
cost_seconds = ((attr_dict['to_time'] - attr_dict['from_time']) / 1000)
print 'cost_seconds ' + str(cost_seconds)
display_time_end = display_time_begin + cost_seconds
print 'display_time_end ' + str(display_time_end)
h_li_array = hxs.select('//body/div[@id="bodydiv"]/div[@class="main_content"]/div/div[@class="left_cont"]/div[@id="msnew"]/div[@class="msnew_infobar"]/ul[@id="ms_tuanlist"]/li')
print "len " + str(len(h_li_array))
xpath_list = [
['img_url', 'div[@class="pic"]/a/img/@src', 'string', None],
['title', 'div[@class="info"]/span/a[@target="_blank"]/@title', 'string', None],
['url', 'div[@class="info"]/span/a[@target="_blank"]/@href', 'string', None],
['origin_price', 'div[@class="info"]/span[@class="gray6"]/text()', 'get_float_str_to_fen', None],
['current_price', 'div[@class="info"]/span[@class="redf16"]/text()', 'get_float_str_to_fen', None],
['sale_info', 'div[@class="info"]/span[@class="red94"]/text()', 'string', None],
]
ret_items = []
for h_li in h_li_array:
attr_dict = get_attr(xpath_list, h_li)
if attr_dict['url'][0] == '/':
attr_dict['url'] = 'http://tuan.vancl.com' + attr_dict['url']
limit = get_num(attr_dict['sale_info'].split(' ')[0])
left = get_num(attr_dict['sale_info'].split(' ')[1])
sale = limit - left
print 'limit ' + str(limit) + " left " + str(left)
prod = VanclMiaoshaItem()
prod['link'] = attr_dict['url']
prod['id'] = hashlib.md5(prod['link']).hexdigest().upper()
prod['title'] = attr_dict['title']
prod['img'] = attr_dict['img_url']
prod['ori_price'] = attr_dict['origin_price']
prod['cur_price'] = attr_dict['current_price']
prod['discount'] = get_discount(attr_dict['origin_price'], attr_dict['current_price'])
#TODO
prod['stat'] = utils.BEGIN
prod['sale'] = sale
prod['sale_percent'] = sale * 100 / limit
prod['display_time_begin'] = display_time_begin
prod['display_time_end'] = display_time_end
#prod['actual_time_begin'] = start_time
#prod['actual_time_end'] = start_time
prod['limit'] = limit
prod['source'] = self.display_name
ret_items.append(prod)
return ret_items
