def prices(code,
startdate=(datetime.datetime.now() -
relativedelta(years=2)).strftime("%Y%m%d"),
enddate=datetime.datetime.now().strftime("%Y%m%d")):
url = "http://finance.sina.com.cn/fund/api/xh5Fund/nav/%s.js" % code
resp = HttpUtil.get(url)
# 将正则表达式编译成Pattern对象
# 查找形似{...}的内容
pattern = re.compile(r'\{.*?\}')
# 返回集合
matches = pattern.findall(resp)
results = collections.OrderedDict()
if matches:
# 使用Match获得分组信息
obj = HttpUtil.parseJson(matches[0])
data = obj["data"]
datas = data.split("#")
for tmp in datas:
values = tmp.split(",")
if startdate <= values[0] <= enddate:
results[datetime.datetime.strptime(
values[0], '%Y%m%d').date()] = float(values[2])
# print("时间：%s 净值：%s 累计净值:%s "%(values[0],values[1],values[2]))
results = collections.OrderedDict(reversed(list(results.items())))
return results
def getFundsByTopic(tp):
pattern = re.compile(r'[a-z0-9]{16}')
if not pattern.match(tp):
for topic in topics():
if topic[1] == tp:
tp = topic[0]
break
url = 'http://fund.eastmoney.com/api/FundTopicInterface.ashx'
params = {
'callbackname': 'topicFundData',
'sort': 'SYL_6Y',
'sorttype': 'DESC',
'pageindex': 1,
'pagesize': 10,
'dt': 10,
'tp': tp
}
json = HttpUtil.get(url, params)
pattern = re.compile(r'\{.*\}')
# 返回集合
matches = pattern.findall(json)
funds = []
if matches:
obj = HttpUtil.parseJson(matches[0])
for item in obj["Datas"]:
fund = Fund(item["FCODE"])
fund.name = item["SHORTNAME"]
funds.append(fund)
else:
logging.error("%s返回异常" % url)
return funds
def get(self):
try:
HttpUtil.validate_ip(self.request)
# 只检查参数,不作业务逻辑处理
self.name = self._check_argument("name", expect_types=(str, unicode))
if self.name == "bufferx":
self.api_response({"e_code": ECODE.SUCCESS, "e_msg": u"Hello, %s!" % self.name})
else:
pass
except HTTPError, e:
g_logger.error(e)
return self.api_response({"e_code": ECODE.HTTP, "e_msg": "%s" % e})
def updateByCateCode(self,
cate_code,
parent_code,
cate_name,
cate_logo,
show_type=TYPE_SHOW_NORMAL):
loginInfo = self.loginHttp.getLoginInfoDict()
if loginInfo is None or loginInfo.keys() is None:
return False
user_tel = loginInfo.keys()[0]
sms_pwd = loginInfo[user_tel]
params = {
"tel": user_tel,
"sms": sms_pwd,
"catecode": cate_code,
"parentcode": parent_code,
"catename": cate_name,
"catelogo": cate_logo,
"showtype": show_type
}
body = HttpUtil.http_post(HttpApi.HOST_URl + HttpApi.URL_UPDATE_CATE,
params=params,
header={})
body = json.loads(body)
print 'updateCate: ', body
if body['code'] == ResponseCode.op_success:
print 'success: ', body['result']
def get(self, *args, **kwargs):
jsCode = self.get_argument('jsCode')
nickName = self.get_argument('nickName')
sign = self.get_argument('sign')
time = self.get_argument('time')
md5Util = MD5Util()
if sign == md5Util.md5Signature(time):
logging = LogUtil().getLogging()
# logging.info('----> jsCode: ' + jsCode)
# logging.info('----> nickName: ' + nickName)
# logging.info('----> sign: ' + sign)
# logging.info('----> time: ' + time)
httpUrl = 'https://api.weixin.qq.com/sns/jscode2session'
param = {
"appid": WxToken.APP_ID,
"secret": WxToken.APP_SECRET,
"js_code": str(jsCode),
"grant_type": 'authorization_code'
}
body = HttpUtil.http_get(httpUrl, params=param)
jsonBody = json.loads(body, "utf8")
if isinstance(jsonBody, dict):
if jsonBody.has_key('openid'):
jsonBody['result'] = True
else:
jsonBody['result'] = False
# logging.info(type(jsonBody))
logging.info('--->session json: ' + str(jsonBody))
else:
jsonBody = json.loads(u'校验失败', "utf8")
self.write(jsonBody)
def __init__(self):
self.dao = CrawlDao.CrawlDao()
self.http = HttpUtil.HttpUtil()
self.brand_dict = dict()
test_header_url = 'http://app.360cec.com'
online_header_url = 'http://10.162.51.140'
self.header_url = test_header_url
try:
url = self.header_url+'/car/info?pid=0'
result = self.http.http_get(url)
json_result = json.loads(result)
alist = json_result['data'][0]['list']
for brand_dic in alist:
brand_name = str(brand_dic['name']).replace("-", "")
brand_id = str(brand_dic['id'])
self.brand_dict[brand_name] = brand_id
# 存车系的name和id
self.series_dict = dict()
for brand_id in self.brand_dict.itervalues():
series_result = self.http.http_get(self.header_url+'/car/info?pid='+brand_id)
json_result = json.loads(series_result)
company_array = json_result['data']
company_dict = dict()
for company in company_array:
company_name = company['tagName']
series_dic = dict()
for series in company['list']:
series_name = series['carName']
series_id = str(series['id'])
series_dic[series_name] = series_id
company_dict[company_name] = series_dic
self.series_dict[brand_id] = company_dict
except Exception as e:
print "==================get error ================%e" % e.message
def __init__(self):
self.dao = None
self.init_dao()
self.http = HttpUtil.HttpUtil()
self.brand_url = "http://auto.jd.com/queryBrands"
self.series_url = "http://auto.jd.com/querySeries?brand=BRAND_VALUE"
self.model_url = "http://auto.jd.com/queryModel?brand=BRAND_VALUE&series=SERIES_VALUE"
# 保养url
self.maintain_url = "http://auto.jd.com/maintain/getMaintain?carModelId=MODEL_ID&mileage=1"
# ===========数据的保存===============
self.max_save_num = 3000
self.jd_car_data_list = list()
self.jd_car_maintain_data_list = list()
self.jd_car_maintain_relation_data_list = list()
self.delete_jd_car_maintain_relation_id_list = list()
# =============数据初始化===================
# 京东车型
self.car_dict = dict()
car_array = self.dao.db.get_data(
"select car_uuid,jd_car_id from jd_car")
for car_data in car_array:
self.car_dict[str(car_data['jd_car_id'])] = str(
car_data['car_uuid'])
def initialize(self, _db):
if __debug__:
g_logger.debug('call initialize()')
self._db = _db
if __debug__:
g_logger.debug(HttpUtil.get_header_string(self.request))
pass
def morejoke(req, lastid):
jokes = HttpUtil.getJokes(lastid)
lastid = jokes[-1]['id']
if req.is_ajax():
t = loader.get_template('ajax_joke.html')
html = t.render(Context({'jokes': jokes}))
return HttpResponse(html)
return render_to_response('joke.html', {'jokes': jokes, 'lastid': lastid,'pagetype':'joke'})
def get(self):
try:
HttpUtil.validate_ip(self.request)
# 只检查参数,不作业务逻辑处理
self.name = self._check_argument('name',
expect_types=(str, unicode))
if self.name == 'bufferx':
self.api_response({
'e_code': ECODE.SUCCESS,
'e_msg': u'Hello, %s!' % self.name
})
else:
pass
except HTTPError, e:
g_logger.error(e)
return self.api_response({'e_code': ECODE.HTTP, 'e_msg': '%s' % e})
def run():
user_agent = HttpUtil.getUserAgent()
headers = {'User-agent': user_agent}
proxy_url = "115.46.77.103:8123"
proxies = {
'http': "http://" + proxy_url,
'https:': "https://" + proxy_url
}
url = "http://www.baidu.com"
res = requests.get(url, headers=headers, proxies=proxies)
print(res.status_code)
def test_html():
# ip = r'192.168.1.62:8280'
# ip1 = 'localhost:8080'
# url = 'http://192.168.1.62:8080/muleApi/api/mule/v1/user/modify_personal_info'
url = 'http://www.baidu.com'
hc = HttpUtil.SimpleHttpClient(url)
hs = hc.execute()
if hs.reason is True and hs.status is 200:
print(hs.data)
else:
print('http request failure, %s,%s' % (hs.status, hs.message))
def valuation(code):
# 估算的url
url_valuation = "http://fundex2.eastmoney.com/FundMobileApi/FundVarietieValuationDetail.ashx"
params = {
"FCODE": code,
"deviceid": "8561658B-9306-4642-A86D-6CAFD29E9C90",
"plat": "iPhone",
"product": "EFund",
"version": "4.2.3"
}
response = HttpUtil.get(url_valuation, params)
obj = HttpUtil.parseJson(response)
try:
gz = float(obj["Expansion"]["GZ"])
except:
gz = 1000
time = obj["Expansion"]["GZTIME"]
zd = obj["Expansion"]["GSZZL"]
# 时间，估值，涨跌
return (time, gz, zd)
def albumvideo(req,aid,page):
videos = HttpUtil.getAlbumVideoAndBeauty(aid,page)
pre = 0
page = int(page)
if page > 0:
pre = page - 1
page += 1
more = 1
if len(videos) == 0:
page -= 2
more = 0
return render_to_response('albumvideo.html', {'videos': videos, 'next': page,'pre':pre, 'aid': aid,'more':more,'pagetype':'album'})
def do_in_action(self):
url = 'https://bkcln.mo9.com/genieApi/api/genie/v1/coupon/lucky_draw'
method = 'POST'
data = {"mobile": self.mobile}
headers = {'Content-Type': "application/json"}
hc = HttpUtil.SimpleHttpClient(url, method, headers, data)
hs = hc.execute()
if hs.reason is True and hs.status is 200:
print(hs.data)
else:
print('http request failure, %s,%s' % (hs.status, hs.message))
return ""
def persistValues(code):
url = "http://finance.sina.com.cn/fund/api/xh5Fund/nav/%s.js" % code
resp = HttpUtil.get(url)
import re
# 将正则表达式编译成Pattern对象
pattern = re.compile(r'\{.*?\}')
# 使用Pattern匹配文本，获得匹配结果，无法匹配时将返回None
ress = pattern.findall(resp)
results = collections.OrderedDict()
if ress:
# 使用Match获得分组信息
obj = HttpUtil.parseJson(ress[0])
data = obj["data"]
datas = data.split("#")
for tmp in datas:
values = tmp.split(",")
results[datetime.datetime.strptime(values[0],
'%Y%m%d')] = float(values[1])
# print("时间：%s 净值：%s 累计净值:%s "%(values[0],values[1],values[2]))
results = collections.OrderedDict(reversed(list(results.items())))
return results
def deleteAdvert(self, advert_id):
loginInfo = self.loginHttp.getLoginInfoDict()
if loginInfo is None or loginInfo.keys() is None:
return False
user_tel = loginInfo.keys()[0]
sms_pwd = loginInfo[user_tel]
params = {"tel": user_tel, "sms": sms_pwd, "adverts_id": advert_id}
body = HttpUtil.http_post(HttpApi.HOST_URl + HttpApi.URL_DELETE_ADVERT, params=params, header={})
body = json.loads(body)
print 'deleteAdvert: ', body
if body['code'] == ResponseCode.op_success:
print 'success: ', body['result']
def getFundsByType(size=100, type=0, page=1):
HttpUtil.setMobileAgent(
"app-iphone-client-(null)-2F1582A1-E089-4446-A561-3D695610241C")
url_funds = "http://fundex2.eastmoney.com/FundMobileApi/FundRankNewList.ashx"
params = {
"FundType": type,
"SortColumn": "RZDF",
"Sort": "desc",
"pageIndex": page,
"pageSize": size,
"BUY": "true",
"CompanyId": "",
"deviceid": "8561658B-9306-4642-A86D-6CAFD29E9C90",
"plat": "iPhone",
"product": "EFund",
"version": "4.2.3"
}
response = HttpUtil.get(url_funds, params)
obj = HttpUtil.parseJson(response)
funds = []
for fund in obj["Datas"]:
funds.append(Fund(fund["FCODE"]))
funds[-1].name = fund["SHORTNAME"]
HttpUtil.setDefaultAgent()
return funds
def addGoods(self, netGoods):
if not netGoods:
return False
loginInfo = self.loginHttp.getLoginInfoDict()
if loginInfo is None or loginInfo.keys() is None:
return False
user_tel = loginInfo.keys()[0]
sms_pwd = loginInfo[user_tel]
# photo_thum_list
goodsPhotoThumList = []
if netGoods.goods_photos_thum_list:
for photoThum in netGoods.goods_photos_thum_list:
string = {
'photos': photoThum.photo,
'thum_photo': photoThum.thum_photo
}
goodsPhotoThumList.append(string)
# size_color_list
goodsSizeColorList = []
if netGoods.attr_size_color_list:
for sizeColor in netGoods.attr_size_color_list:
string = {
'goodssize': sizeColor.attr_size,
'goodscolor': sizeColor.attr_color
}
goodsSizeColorList.append(string)
params = {
"tel": user_tel,
"sms": sms_pwd,
"cateid": netGoods.cate_id,
"brandid": netGoods.brand_id,
"name": netGoods.goods_name,
"marketprice": netGoods.market_price,
"currentprice": netGoods.current_price,
"salecount": netGoods.sale_count,
"stocknum": netGoods.stock_num,
"status": netGoods.status,
"goodscode": netGoods.goods_code,
"goodslogo": netGoods.goods_logo,
"thumlogo": netGoods.thum_logo,
"keywords": netGoods.keywords,
"photosthumlist": goodsPhotoThumList,
"marketyear": netGoods.attr_market_year,
"sizecolorlist": goodsSizeColorList
}
body = HttpUtil.http_post(HttpApi.HOST_URl + HttpApi.URL_ADD_GOODS,
params=params,
header={})
body = json.loads(body)
print 'addGoods: ', body
if body['code'] == ResponseCode.op_success:
print 'success: ', body['result']
def addAdvert(self, cate_id, advert_title, advert_sort, advert_pic_url):
loginInfo = self.loginHttp.getLoginInfoDict()
if loginInfo is None or loginInfo.keys() is None:
return False
user_tel = loginInfo.keys()[0]
sms_pwd = loginInfo[user_tel]
params = {"tel": user_tel, "sms": sms_pwd, "cate_id": cate_id, "title": advert_title,
"sort": advert_sort, "pic_url": advert_pic_url}
body = HttpUtil.http_post(HttpApi.HOST_URl + HttpApi.URL_ADD_ADVERT, params=params, header={})
body = json.loads(body)
print 'addAdvert: ', body
if body['code'] == ResponseCode.op_success:
print 'success: ', body['result']
def statistics_search():
# url = 'http://localhost:8080/bk/api/genie/v1/statistics/search'
# url = 'https://bkcln.mo9.com/genieApi/bk/api/genie/v1/statistics/search'
url = 'https://bkcln.mo9.com/genieApi/bk/api/genie/v1/statistics/search'
method = 'GET'
headers = {
'Ticket': 'sss',
'User-Id': "274"
}
hc = HttpUtil.SimpleHttpClient(url, method, headers)
hs = hc.execute()
if hs.reason is True and hs.status is 200:
print(hs.data)
else:
print('http request failure, %s,%s' % (hs.status, hs.message))
def topics():
url = 'http://fund.eastmoney.com/api/FundTopicInterface.ashx'
params = {
"callbackname": "fundData",
"sort": "SYL_Z",
'sorttype': 'desc',
'pageindex': 1,
'pagesize': 500,
'dt': 11,
'tt': 0,
'rs': 'WRANK'
}
json = HttpUtil.get(url, params)
pattern = re.compile(r'\{.*?\}')
# 返回集合
matches = pattern.findall(json)
topics = []
if matches:
obj = HttpUtil.parseJson(matches[0])
for item in obj["Datas"]:
topics.append(item.split(","))
return topics
else:
logging.error("%s返回异常" % url)
def albumbeauty(req,aid,page):
albums = HttpUtil.getAlbumVideoAndBeauty(aid, page)
pre = 0
page = int(page)
if page > 0:
pre = page - 1
page += 1
more = 1
if len(albums) == 0:
page -= 2
more = 0
if req.is_ajax():
t = loader.get_template('ajax_album_beauty.html')
html = t.render(Context({'albums': albums}))
return HttpResponse(html)
return render_to_response('albumbeauty.html', {'albums': albums, 'next': page,'pre':pre, 'aid': aid,'more':more,'pagetype':'album'})
def login(self, user_tel, sms_pwd):
params = {"tel": user_tel, "sms": sms_pwd}
# header = {'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0'}
body = HttpUtil.http_post(HttpApi.HOST_URl + HttpApi.URL_LOGIN,
params=params,
header={})
body = json.loads(body)
print 'login: '******'code'] == ResponseCode.op_success:
print 'success: ', body['result']
# save login info dict
GlobalVar.setValue(self.global_key_tel, user_tel)
GlobalVar.setValue(self.global_value_pwd, sms_pwd)
return True
else:
return False
def modify_user_info():
# ip = r'192.168.1.62:8280'
# ip1 = 'localhost:8080'
# url = 'http://192.168.1.62:8080/muleApi/api/mule/v1/user/modify_personal_info'
url = 'http://localhost:8080/api/mule/v1/user/modify_personal_info'
method = 'POST'
access_token = '2c7805d64d2e84fc14e2cd8bca91e0d8d909774d'
user_id = '78'
data = {'birthday': get_timestamp()}
headers = {
'Access-Token': access_token,
'Current-User-Id': user_id,
'Content-Type': "application/json"
}
hc = HttpUtil.SimpleHttpClient(url, method, headers, data)
hs = hc.execute()
if hs.reason is True and hs.status is 200:
print(hs.data)
else:
print('http request failure, %s,%s' % (hs.status, hs.message))
def order_callback():
# ip = r'192.168.1.62:8280'
# ip1 = 'localhost:8080'
# url = 'http://192.168.1.62:8080/muleApi/api/mule/v1/user/modify_personal_info'
url = 'http://localhost:8080/api/mule/v1/member/order_callback'
method = 'POST'
post_data = {"amount": "0.01", # 金额
"topuporderDealcode": "WDAEQCFONCORCOAD", # 第三方订单号
"channel": "yilianh5", # 第三方渠道
"dealcode": "150408407489400983", # 交易订单号
"status": "success"} # 交易状态
post_data_str = json.dumps(post_data)
sign_data = str('postData=' + post_data_str + "643138394F10DA5E9647709A3FA8DD7F")
sign = hashlib.md5(sign_data.encode("utf8"))
sign_data = str('postData=' + post_data_str + "&") + str('sign=' + sign.hexdigest() + "&")
url = url + '?' + sign_data
hc = HttpUtil.SimpleHttpClient(url, method)
hs = hc.execute()
if hs.reason is True and hs.status is 200:
print(hs.data)
else:
print('http request failure, %s,%s' % (hs.status, hs.message))
def getFundsByDT(size=100, page=1):
HttpUtil.setMobileAgent()
url_funds = "http://fundex2.eastmoney.com/FundMobileApi/FundInvestmentRankList.ashx"
params = {
"FundType": "0",
"SortColumn": "DT_1N",
"Sort": "desc",
"pageIndex": page,
"pageSize": size,
"BUY": "true",
"CompanyId": "",
"deviceid": "8561658B-9306-4642-A86D-6CAFD29E9C90",
"plat": "iPhone",
"product": "EFund",
"version": "4.2.3"
}
response = HttpUtil.get(url_funds, params)
obj = HttpUtil.parseJson(response)
funds = []
for fund in obj["Datas"]:
funds.append(Fund(fund["FCODE"]))
funds[-1].name = fund["SHORTNAME"]
HttpUtil.setDefaultAgent()
return funds
def initialize(self):
if __debug__:
g_logger.debug('call initialize()')
self.set_header('Server', BaseHandler.HTTP_SERVER_NAME)
g_logger.info(HttpUtil.get_header_string(self.request))
pass
def initialize(self, version):
self.version = version
if __debug__:
g_logger.debug(HttpUtil.get_header_string(self.request))
def initialize(self):
if __debug__:
g_logger.debug('call initialize()')
self.set_header('Server', BaseHandler.HTTP_SERVER_NAME)
g_logger.info(HttpUtil.get_header_string(self.request))
pass
def initialize(self, version):
self.version = version
if __debug__:
g_logger.debug(HttpUtil.get_header_string(self.request))
def joke(req):
jokes = HttpUtil.getJokes('-1')
lastid = jokes[-1]['id']
return render_to_response('joke.html', {'jokes':jokes,'lastid': lastid,'pagetype':'joke'})
def morealbum(req,lastid):
albums = HttpUtil.getAlbums(lastid)
lastid = albums[-1]['id']
return render_to_response('album.html', {'albums':albums,'lastid':lastid,'pagetype':'album'})
